diff --git a/.changeset/beige-glasses-dream.md b/.changeset/beige-glasses-dream.md new file mode 100644 index 0000000000..c1a9ad1857 --- /dev/null +++ b/.changeset/beige-glasses-dream.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-scaffolder': patch +--- + +Allow to pass custom TemplateCard to ScaffolderPage diff --git a/.changeset/cuddly-cooks-fry.md b/.changeset/cuddly-cooks-fry.md new file mode 100644 index 0000000000..028b0291bd --- /dev/null +++ b/.changeset/cuddly-cooks-fry.md @@ -0,0 +1,6 @@ +--- +'@backstage/plugin-catalog-react': patch +--- + +Added a `useEntityKinds` hook to load a unique list of entity kinds from the catalog. +Fixed a bug in `EntityTypePicker` where the component did not hide when no types were available in returned entities. diff --git a/.changeset/cuddly-impalas-wave.md b/.changeset/cuddly-impalas-wave.md new file mode 100644 index 0000000000..1199ef1482 --- /dev/null +++ b/.changeset/cuddly-impalas-wave.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +GitHub discovery processor passes over repositories that do not have a default branch diff --git a/.changeset/fair-files-rest.md b/.changeset/fair-files-rest.md new file mode 100644 index 0000000000..68d3246aa4 --- /dev/null +++ b/.changeset/fair-files-rest.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-scaffolder-backend': patch +--- + +GitHub create repository webhook action: `github:webhook` for Backstage plugin Scaffolder has been added. diff --git a/.changeset/forty-terms-join.md b/.changeset/forty-terms-join.md new file mode 100644 index 0000000000..1476ebff9c --- /dev/null +++ b/.changeset/forty-terms-join.md @@ -0,0 +1,9 @@ +--- +'@backstage/plugin-techdocs': patch +'@backstage/plugin-techdocs-backend': patch +'@backstage/techdocs-common': patch +--- + +Adding in-context search to TechDocs Reader component. Using existing search-backend to query for indexed search results scoped into a specific entity's techdocs. Needs TechDocsCollator enabled on the backend to work. + +Adding extra information to indexed tech docs documents for search. diff --git a/.changeset/funny-pants-unite.md b/.changeset/funny-pants-unite.md deleted file mode 100644 index 349cf54b8f..0000000000 --- a/.changeset/funny-pants-unite.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@backstage/plugin-catalog-import': patch ---- - -Fix importing yaml files from URLs with trailing query parameters. diff --git a/.changeset/good-papayas-fetch.md b/.changeset/good-papayas-fetch.md new file mode 100644 index 0000000000..6cd74ce064 --- /dev/null +++ b/.changeset/good-papayas-fetch.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +Adds optional namespacing for users in the GitHub Multi Org Plugin diff --git a/.changeset/hip-beers-attend.md b/.changeset/hip-beers-attend.md new file mode 100644 index 0000000000..802bfe7397 --- /dev/null +++ b/.changeset/hip-beers-attend.md @@ -0,0 +1,7 @@ +--- +'@backstage/core-components': patch +'@backstage/plugin-catalog-import': patch +'@backstage/plugin-shortcuts': patch +--- + +Bumped `react-hook-form` to `^7.12.2` diff --git a/.changeset/honest-suits-approve.md b/.changeset/honest-suits-approve.md new file mode 100644 index 0000000000..59cdb09762 --- /dev/null +++ b/.changeset/honest-suits-approve.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-git-release-manager': patch +--- + +Wrap each feature in custom feature arrays with an element containing a 'key' to avoid missing-key-warnings diff --git a/.changeset/nasty-ads-nail.md b/.changeset/nasty-ads-nail.md new file mode 100644 index 0000000000..0518b1e569 --- /dev/null +++ b/.changeset/nasty-ads-nail.md @@ -0,0 +1,10 @@ +--- +'@backstage/core-components': minor +--- + +Changed the `titleComponent` prop on `ContentHeader` to accept `ReactNode` instead of a React `ComponentType`. Usages of this prop should be converted from passing a component to passing in the rendered element: + +```diff +- ++}> +``` diff --git a/.changeset/nasty-jokes-camp.md b/.changeset/nasty-jokes-camp.md new file mode 100644 index 0000000000..e30ff75911 --- /dev/null +++ b/.changeset/nasty-jokes-camp.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-org': patch +--- + +Don't open a new tab when clicking on the ownership card. diff --git a/.changeset/new-ligers-drum.md b/.changeset/new-ligers-drum.md deleted file mode 100644 index 0edee17b4d..0000000000 --- a/.changeset/new-ligers-drum.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@backstage/plugin-xcmetrics': patch ---- - -Enable browsing detailed build information such as host configuration, errors, warnings, metadata and a timeline for all targets diff --git a/.changeset/pretty-cameras-give.md b/.changeset/pretty-cameras-give.md new file mode 100644 index 0000000000..8f9f4a83fa --- /dev/null +++ b/.changeset/pretty-cameras-give.md @@ -0,0 +1,7 @@ +--- +'@backstage/plugin-catalog': patch +--- + +Update the `AboutCard` to properly support non-standard entity types and rework the defaults for the build-in kinds. + +This change also uses `useElementFilter(...)` instead of `React.children.count(...)` in `AboutField` to properly recognize whether children are available. diff --git a/.changeset/quick-trains-flow.md b/.changeset/quick-trains-flow.md deleted file mode 100644 index 447dc0aacc..0000000000 --- a/.changeset/quick-trains-flow.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@backstage/plugin-jenkins': patch ---- - -Make `isJenkinsAvailable` recognise both the old and the new Jenkins annotation. diff --git a/.changeset/serious-maps-try.md b/.changeset/serious-maps-try.md new file mode 100644 index 0000000000..78e953dfd6 --- /dev/null +++ b/.changeset/serious-maps-try.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-auth-backend': patch +--- + +typo fix `tenentId` in Azure auth provider docs diff --git a/.changeset/short-years-smile.md b/.changeset/short-years-smile.md new file mode 100644 index 0000000000..6e7b882c1e --- /dev/null +++ b/.changeset/short-years-smile.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog': patch +--- + +Added the ability to switch entity kind on the catalog index page. This is a non-breaking change, but if you created a custom `CatalogPage` and wish to use this feature, make the modifications shown on [#6895](https://github.com/backstage/backstage/pull/6895). diff --git a/.changeset/sixty-cougars-study.md b/.changeset/sixty-cougars-study.md new file mode 100644 index 0000000000..80db2913d0 --- /dev/null +++ b/.changeset/sixty-cougars-study.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-shortcuts': patch +--- + +Fix material-ui warning about deprecated `onExit` prop diff --git a/.changeset/strange-zoos-grab.md b/.changeset/strange-zoos-grab.md new file mode 100644 index 0000000000..78a6c6eeee --- /dev/null +++ b/.changeset/strange-zoos-grab.md @@ -0,0 +1,11 @@ +--- +'@backstage/catalog-model': patch +--- + +Add an optional `metadata.title` field to all entity kinds. + +This used to be available on only the `Template` kind, and we have decided that the metadata block should be the same for all kinds. A title can be useful especially in large and complex catalogs where users have a tough time navigating or discerning among the entities. + +It also carries some risk. You do not want to end up giving a title that collides with an actual name, which at best leads to confusion and at worst could be a liability. We do not perform any collision detection in the catalog. If you want to disallow this facility you may want to add a small processor that makes sure it's not set. + +At the time of writing this message, only the scaffolder actually makes use of this field for display purposes. diff --git a/.changeset/strong-beers-ring.md b/.changeset/strong-beers-ring.md new file mode 100644 index 0000000000..17a6d89fcc --- /dev/null +++ b/.changeset/strong-beers-ring.md @@ -0,0 +1,5 @@ +--- +'@backstage/create-app': patch +--- + +Minor tweaks to the create-app template to match earlier documented changes diff --git a/.changeset/stupid-ants-call.md b/.changeset/stupid-ants-call.md new file mode 100644 index 0000000000..b2dc16d0e2 --- /dev/null +++ b/.changeset/stupid-ants-call.md @@ -0,0 +1,5 @@ +--- +'@backstage/backend-common': patch +--- + +Export type that are needed to implement a new `UrlReader` diff --git a/.changeset/sweet-jars-travel.md b/.changeset/sweet-jars-travel.md new file mode 100644 index 0000000000..83fcd13623 --- /dev/null +++ b/.changeset/sweet-jars-travel.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-sonarqube': patch +--- + +Update README to show a simpler proxy configuration. diff --git a/.changeset/techdocs-trains-beg.md b/.changeset/techdocs-trains-beg.md new file mode 100644 index 0000000000..05161b7093 --- /dev/null +++ b/.changeset/techdocs-trains-beg.md @@ -0,0 +1,5 @@ +--- +'@backstage/techdocs-common': patch +--- + +Force using `posix` path for cloud storage diff --git a/.changeset/thick-trees-pay.md b/.changeset/thick-trees-pay.md new file mode 100644 index 0000000000..6ee7b514ca --- /dev/null +++ b/.changeset/thick-trees-pay.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog': patch +--- + +Use a `Link` for the edit button on the `AboutCard` instead of doing `window.open(...)` diff --git a/.changeset/wicked-singers-push.md b/.changeset/wicked-singers-push.md new file mode 100644 index 0000000000..3c34a959ed --- /dev/null +++ b/.changeset/wicked-singers-push.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-org': patch +--- + +Fix OwnershipCard links to support namespaced owners diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index fda500ed02..781feb1f04 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -17,6 +17,7 @@ /plugins/techdocs @backstage/techdocs-core /plugins/techdocs-backend @backstage/techdocs-core /plugins/ilert @yacut +/plugins/home @backstage/techdocs-core /packages/search-common @backstage/techdocs-core /packages/techdocs-common @backstage/techdocs-core /.changeset/cost-insights-* @backstage/silver-lining diff --git a/.github/styles/vocab.txt b/.github/styles/vocab.txt index 57510f2c3f..dd9224fc84 100644 --- a/.github/styles/vocab.txt +++ b/.github/styles/vocab.txt @@ -142,6 +142,7 @@ maintainership makefile md memcache +memoize memoized microservice microservices @@ -215,6 +216,7 @@ repo Repo repos rerender +rerenders Reusability reusability roadmaps diff --git a/.github/workflows/chromatic-storybook-test.yml b/.github/workflows/chromatic-storybook-test.yml index 043777601f..4c64db309d 100644 --- a/.github/workflows/chromatic-storybook-test.yml +++ b/.github/workflows/chromatic-storybook-test.yml @@ -5,6 +5,7 @@ on: - '.github/workflows/chromatic-storybook-test.yml' - 'packages/storybook/**' - 'packages/core-components/src/**' + - '**/*.stories.tsx' jobs: chromatic: diff --git a/ADOPTERS.md b/ADOPTERS.md index 72b6cce072..18d1388936 100644 --- a/ADOPTERS.md +++ b/ADOPTERS.md @@ -21,7 +21,7 @@ | [Paddle.com](https://paddle.com) | [Ioannis Georgoulas](https://github.com/geototti21) | Developer portal (Tech Docs, Service Catalog, Internal Tooling), we use vanilla Backstage FE and custom BE implementation in Go | | [Acast.com](https://acast.com) | [Olle Lundberg](https://github.com/lndbrg) | Developer portal with tech docs, service catalog and a bunch of other internal tooling | | [Lunar](https://lunar.app) | [Jacob Valdemar](https://github.com/JacobValdemar) | Internal developer portal for service overview and insights, API documentation, technical guides, onboarding guides and RFC's. | -| [Trendyol](https://trendyol.com) | [Erdogan Oksuz](https://github.com/erdoganoksuz) | The Developer Portal has been called `Pandora`. Provides an overview of Trendyol tech ecosystem. TechDocs, Catalog, Custom Plugins and Theme. | +| [Trendyol](https://trendyol.com) | [Gamze Senturk](https://github.com/gmzsenturk), [Mert Can Bilgic](https://github.com/mertcb) | The Developer Portal has been called `Pandora`. Provides an overview of Trendyol tech ecosystem. TechDocs, Catalog, Custom Plugins and Theme. | | [Peloton](https://www.onepeloton.com/) | [Jim Haughwout](https://github.com/JimHaughwout) | Creating our first developer portal and tech-docs. Exploring Service Catalog, Tech Insights and Cost Insights as well. | | [TELUS](https://telus.com) | [Seb Barre](https://github.com/sbarre) | The Go-to place to find answers about development and delivery at TELUS. | | [Brex](https://www.brex.com/) | [Vamsi Chitters](https://github.com/vamsikc) | A centralized UI to understand how a service fits in the whole Brex architecture and manage a team’s engineering dependencies. | @@ -42,3 +42,6 @@ | [empathy.co](https://empathy.co/) | [@guillermotti](https://github.com/guillermotti) | Developer portal for tech docs, service catalog, plugin discovery and much more. | | [creditas.com](https://creditas.com/) | [@aureliosaraiva](https://github.com/aureliosaraiva) [@Creditas](https://github.com/creditas) | Centralization of all services, standards, documentation, etc. We started the deployment process. | | [Prisjakt](https://www.prisjakt.nu) / [PriceSpy](https://pricespy.co.uk) | [@kennylindahl](https://github.com/kennylindahl) | Internal developer portal - Documentation, scaffolding, software catalog, TechRadar, Gitlab org data integration | +| [Powerspike](https://powerspike.tv/) | [@trelore](https://github.com/trelore) | Developer portal for documentation of core libraries and repositories. | +| [2U](https://2u.com) | [Andrew Thal](https://github.com/athal7) | Development team home-base, promoting service discoverability, resource dependencies, and tech radar | +| [Taxfix](https://taxfix.de/) | [Sami Ur Rehman](https://github.com/samiurrehman92) | Developer's portal with software catalog at it's core. Hosts API Specs, Tech Docs, Tech Radar and some custom plugins. | diff --git a/app-config.yaml b/app-config.yaml index c1d0121dff..9f365becef 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -160,9 +160,13 @@ integrations: azure: - host: dev.azure.com token: ${AZURE_TOKEN} -# googleGcs: -# clientEmail: 'example@example.com' -# privateKey: ${GCS_PRIVATE_KEY} + # googleGcs: + # clientEmail: 'example@example.com' + # privateKey: ${GCS_PRIVATE_KEY} + awsS3: + - host: amazonaws.com + accessKeyId: ${AWS_ACCESS_KEY_ID} + secretAccessKey: ${AWS_SECRET_ACCESS_KEY} catalog: rules: diff --git a/docs/auth/github/provider.md b/docs/auth/github/provider.md index 4b65c8c3c2..81b2e98f2c 100644 --- a/docs/auth/github/provider.md +++ b/docs/auth/github/provider.md @@ -10,11 +10,16 @@ that can authenticate users using GitHub or GitHub Enterprise OAuth. ## Create an OAuth App on GitHub -To add GitHub authentication, you must create an OAuth App from the GitHub +To add GitHub authentication, you must create either a GitHub App, or an OAuth +App from the GitHub [developer settings](https://github.com/settings/developers). The `Homepage URL` should point to Backstage's frontend, while the `Authorization callback URL` will point to the auth backend. +Note that if you're using a GitHub App, the allowed scopes are configured as +part of that app. This means you need to verify what scopes the plugins you use +require, so be sure to check the plugin READMEs for that information. + Settings for local development: - Application name: Backstage (or your custom app name) @@ -51,3 +56,11 @@ The GitHub provider is a structure with three configuration keys: To add the provider to the frontend, add the `githubAuthApi` reference and `SignInPage` component as shown in [Adding the provider to the sign-in page](../index.md#adding-the-provider-to-the-sign-in-page). + +## Difference between GitHub Apps and GitHub OAuth Apps + +GitHub Apps handle OAuth scope at the app installation level, meaning that the +`scope` parameter for the call to `getAccessToken` in the frontend has no +effect. When calling `getAccessToken` in open source plugins, one should still +include the appropriate scope, but also document in the plugin README what +scopes are required for GitHub Apps. diff --git a/docs/auth/identity-resolver.md b/docs/auth/identity-resolver.md index 7ec6ec7117..2c36ddcd7f 100644 --- a/docs/auth/identity-resolver.md +++ b/docs/auth/identity-resolver.md @@ -128,20 +128,20 @@ export default async function createPlugin({ google: createGoogleProvider({ signIn: { resolver: async ({ profile: { email } }, ctx) => { - const [sub] = email?.split('@') ?? ''; + const [id] = email?.split('@') ?? ''; // Fetch from an external system that returns entity claims like: // ['user:default/breanna.davison', ...] const ent = await externalSystemClient.getUsernames(email); // Resolve group membership from the Backstage catalog const fullEnt = await ctx.catalogIdentityClient.resolveCatalogMembership({ - entityRefs: [sub].concat(ent), + entityRefs: [id].concat(ent), logger: ctx.logger, }); const token = await ctx.tokenIssuer.issueToken({ - claims: { sub, ent: fullEnt }, + claims: { sub: id, ent: fullEnt }, }); - return { sub, token }; + return { id, token }; }, }, }), diff --git a/docs/auth/microsoft/provider.md b/docs/auth/microsoft/provider.md index b64f9b1902..52f463bab4 100644 --- a/docs/auth/microsoft/provider.md +++ b/docs/auth/microsoft/provider.md @@ -44,7 +44,7 @@ The Microsoft provider is a structure with three configuration keys: - `clientId`: Application (client) ID, found on App Registration > Overview - `clientSecret`: Secret, found on App Registration > Certificates & secrets -- `tenentId`: Directory (tenant) ID, found on App Registration > Overview +- `tenantId`: Directory (tenant) ID, found on App Registration > Overview ## Adding the provider to the Backstage frontend diff --git a/docs/features/search/search-engines.md b/docs/features/search/search-engines.md index 69a223b29e..a0302ce279 100644 --- a/docs/features/search/search-engines.md +++ b/docs/features/search/search-engines.md @@ -43,7 +43,7 @@ provides decent results and performs well with ten thousands of indexed documents. The connection to postgres is established via the database manager also used by other plugins. -> **Important**: The search plugin requires at least Postgres 11! +> **Important**: The search plugin requires at least Postgres 12! To use the `PgSearchEngine`, make sure that you have a Postgres database configured and make the following changes to your backend: diff --git a/docs/features/software-catalog/catalog-customization.md b/docs/features/software-catalog/catalog-customization.md index 07258a55cb..8f855776a0 100644 --- a/docs/features/software-catalog/catalog-customization.md +++ b/docs/features/software-catalog/catalog-customization.md @@ -34,15 +34,14 @@ export const CustomCatalogPage = ({ }: CatalogPageProps) => { return ( - - - - All your software catalog entities - - + + + }> + + All your software catalog entities + - - - + + ); }; diff --git a/docs/features/software-catalog/descriptor-format.md b/docs/features/software-catalog/descriptor-format.md index f39454a797..c2141db0e5 100644 --- a/docs/features/software-catalog/descriptor-format.md +++ b/docs/features/software-catalog/descriptor-format.md @@ -253,6 +253,22 @@ the entity belongs to the `"default"` namespace. Namespaces may also be part of the catalog, and are `v1` / `Namespace` entities, i.e. not Backstage specific but the same as in Kubernetes. +### `title` [optional] + +A display name of the entity, to be presented in user interfaces instead of the +`name` property above, when available. + +This field is sometimes useful when the `name` is cumbersome or ends up being +perceived as overly technical. The title generally does not have as stringent +format requirements on it, so it may contain special characters and be more +explanatory. Do keep it very short though, and avoid situations where a title +can be confused with the name of another entity, or where two entities share a +title. + +Note that this is only for display purposes, and may be ignored by some parts of +the code. [Entity references](references.md) still always make use of the `name` +property for example, not the title. + ### `description` [optional] A human readable description of the entity, to be shown in Backstage. Should be @@ -698,12 +714,6 @@ shape, this kind has the following structure. Exactly equal to `backstage.io/v1beta2` and `Template`, respectively. -### `metadata.title` [required] - -The nice display name for the template as a string, e.g. `React SSR Template`. -This field is required as is used to reference the template to the user instead -of the `metadata.name` field. - ### `metadata.tags` [optional] A list of strings that can be associated with the template, e.g. diff --git a/docs/features/software-templates/migrating-from-v1alpha1-to-v1beta2.md b/docs/features/software-templates/migrating-from-v1alpha1-to-v1beta2.md index 25ff2a2f3a..28001cce0b 100644 --- a/docs/features/software-templates/migrating-from-v1alpha1-to-v1beta2.md +++ b/docs/features/software-templates/migrating-from-v1alpha1-to-v1beta2.md @@ -325,7 +325,7 @@ spec: output: links: - url: '{{steps.publish.output.remoteUrl}}' - text: 'Go to Repo' + title: 'Go to Repo' ``` ## Questions? diff --git a/docs/features/software-templates/writing-custom-actions.md b/docs/features/software-templates/writing-custom-actions.md index 53a90e2954..a5773a6aff 100644 --- a/docs/features/software-templates/writing-custom-actions.md +++ b/docs/features/software-templates/writing-custom-actions.md @@ -131,6 +131,9 @@ want to have those as well as your new one, you'll need to do the following: ```ts import { createBuiltinActions } from '@backstage/plugin-scaffolder-backend'; +import { ScmIntegrations } from '@backstage/integration'; + +const integrations = ScmIntegrations.fromConfig(config); const builtInActions = createBuiltinActions({ containerRunner, diff --git a/docs/features/techdocs/README.md b/docs/features/techdocs/README.md index 6732b3ab18..b798742463 100644 --- a/docs/features/techdocs/README.md +++ b/docs/features/techdocs/README.md @@ -29,7 +29,7 @@ about TechDocs and the philosophy in its - Explore and take advantage of the large ecosystem of [MkDocs plugins](https://www.mkdocs.org/user-guide/plugins/) to create a rich reading experience. -- Search for and find docs (coming soon). +- Search for and find docs. - Highlight text and raise an Issue to create feedback loop to drive quality documentation (future). - Contribute to and deploy from a marketplace of TechDocs widgets (future). @@ -54,23 +54,12 @@ providers are used. | Google Cloud Storage (GCS) | Yes ✅ | | Amazon Web Services (AWS) S3 | Yes ✅ | | Azure Blob Storage | Yes ✅ | -| OpenStack Swift | Yes ✅ | +| OpenStack Swift | Community ✅ | [Reach out to us](#feedback) if you want to request more platforms. ## Project roadmap -### **Ongoing work 🚧** - -**Beta release** - -[Milestone](https://github.com/backstage/backstage/milestone/29) - -- It should be possible and easy to use TechDocs in most environments across - organizations. -- Minimal bugs, better error handling and scalable backend and frontend. -- Documentation Search -- TechDocs Homepage with basic features - ### **Future work 🔮** **General Availability (GA) release** - diff --git a/docs/features/techdocs/configuration.md b/docs/features/techdocs/configuration.md index 4bb6857106..46cbc2a8a2 100644 --- a/docs/features/techdocs/configuration.md +++ b/docs/features/techdocs/configuration.md @@ -113,6 +113,14 @@ techdocs: # https://docs.microsoft.com/en-us/azure/storage/common/storage-auth?toc=/azure/storage/blobs/toc.json accountKey: ${TECHDOCS_AZURE_BLOB_STORAGE_ACCOUNT_KEY} + # (Optional and not recommended) Prior to version [0.x.y] of TechDocs, docs + # sites could only be accessed over paths with case-sensitive entity triplets + # e.g. (namespace/Kind/name). If you are upgrading from an older version of + # TechDocs and are unable to perform the necessary migration of files in your + # external storage, you can set this value to `true` to temporarily revert to + # the old, case-sensitive entity triplet behavior. + legacyUseCaseSensitiveTripletPaths: false + # (Optional and Legacy) TechDocs makes API calls to techdocs-backend using this URL. e.g. get docs of an entity, get metadata, etc. # You don't have to specify this anymore. diff --git a/docs/features/techdocs/getting-started.md b/docs/features/techdocs/getting-started.md index cf0bbb0de1..0b1809dc22 100644 --- a/docs/features/techdocs/getting-started.md +++ b/docs/features/techdocs/getting-started.md @@ -33,14 +33,24 @@ In `packages/app/src/App.tsx`, import `TechDocsPage` and add the following to `FlatRoutes`: ```tsx -import { TechDocsPage } from '@backstage/plugin-techdocs'; +import { + DefaultTechDocsHome, + TechDocsIndexPage, + TechDocsReaderPage, +} from '@backstage/plugin-techdocs'; // ... const AppRoutes = () => { // ... other plugin routes - } /> + }> + + + } + /> ; }; ``` @@ -204,7 +214,7 @@ techdocs: builder: 'local' publisher: type: 'local' - generators: + generator: techdocs: local ``` diff --git a/docs/features/techdocs/how-to-guides.md b/docs/features/techdocs/how-to-guides.md index e1533498d7..5fa14472d1 100644 --- a/docs/features/techdocs/how-to-guides.md +++ b/docs/features/techdocs/how-to-guides.md @@ -104,83 +104,110 @@ the repository. The archive does not have any git history attached to it. Also it is a compressed file. Hence the file size is significantly smaller than how much data git clone has to transfer. -## How to use a custom TechDocs home page? +## How to customize the TechDocs home page? -### 1st way: TechDocsCustomHome with a custom configuration +TechDocs uses a composability pattern similar to the Search and Catalog plugins +in Backstage. While a default table experience, similar to the one provided by +the Catalog plugin, is made available for ease-of-use, it's possible for you to +provide a completely custom experience, tailored to the needs of your +organization. -As an example, in your main App.tsx: +This is done in your `app` package. By default, you might see something like +this in your `App.tsx`: ```tsx -import { - TechDocsCustomHome, - PanelType, - TechDocsReaderPage, -} from '@backstage/plugin-techdocs'; -import { Entity } from '@backstage/catalog-model'; - -const tabsConfig = [ - { - label: 'Custom Tab', - panels: [ - { - title: 'Custom Documents Cards 1', - description: - 'Explore your internal technical ecosystem through documentation.', - panelType: 'DocsCardGrid' as PanelType, - // optional, is applied to a container of the panel (excludes header of panel) - panelCSS: { maxHeight: '400px', overflow:'auto' }, - filterPredicate: (entity: Entity) => !!entity.metadata.annotations?.['customCardAnnotationOne']; - }, - { - title: 'Custom Documents Cards 2', - description: - 'Explore your internal technical ecosystem through documentation.', - panelType: 'DocsCardGrid' as PanelType, - panelCSS: { maxHeight: '400px', overflow:'auto' }, - filterPredicate: (entity: Entity) => !!entity.metadata.annotations?.['customCardAnnotationTwo']; - }, - ], - }, - { - label: 'Overview', - panels: [ - { - title: 'Overview', - description: - 'Explore your internal technical ecosystem through documentation.', - panelType: 'DocsTable' as PanelType, - filterPredicate: () => true, - }, - ], - }, -]; - -const routes = ( +const AppRoutes = () => { - } - /> - } - /> - + }> + + + ; +}; ``` -An example of tabsConfig that corresponds to the default documentation home page -can be found at `plugins/techdocs/src/home/components/TechDocsHome.tsx`. +But you can replace `` with any React component, which +will be rendered in its place. Most likely, you would want to create and +maintain such a component in a new directory at +`packages/app/src/components/techdocs`, and import and use it in `App.tsx`: -Currently `panelType` has DocsCardGrid and DocsTable available. We currently -recommend that DocsCardGrid can be optionally vertically stacked by setting a -maxHeight using `panelCSS`, and DocsTable to be in a tab by itself. +```tsx +import { CustomTechDocsHome } from './components/techdocs/CustomTechDocsHome'; +// ... +const AppRoutes = () => { + + }> + + + ; +}; +``` -### 2nd way: Custom home page plugin +## How to migrate from TechDocs Alpha to Beta -A custom home page plugin can be built that uses the components extensions -DocsCardGrid and DocsTable, exported from @backstage/techdocs. They both take a -array of documentation entities ( i.e.have a 'backstage.io/techdocs-ref' -annotation ) as an 'entities' attribute. +> This guide only applies to the "recommended" TechDocs deployment method (where +> an external storage provider and external CI/CD is used). If you use the +> "basic" or "out-of-the-box" setup, you can stop here! No action needed. -For a reference to the React structure of the default home page, please refer to -`plugins/techdocs/src/home/components/TechDocsCustomHome.tsx`. +The beta version of TechDocs (v0.x.y) made a breaking change to the way TechDocs +content was accessed and stored, allowing pages to be accessed with +case-insensitive entity triplet paths (e.g. `/docs/namespace/kind/name` whereas +in prior versions, they could only be accessed at `/docs/namespace/Kind/name`). +In order to enable this change, documentation has to be stored in an external +storage provider using an object key whose entity triplet is lower-cased. + +New installations of TechDocs since the beta version will work fine with no +action, but for those who were running TechDocs prior to this version, a +migration will need to be performed so that all existing content in your storage +bucket matches this lower-case entity triplet expectation. + +1. **Ensure you have the right permissions on your storage provider**: In order + to migrate files in your storage provider, the `techdocs-cli` needs to be + able to read/copy/rename/move/delete files. The exact instructions vary by + storage provider, but check the [using cloud storage][using-cloud-storage] + page for details. + +2. **Run a non-destructive migration of files**: Ensure you have the latest + version of `techdocs-cli` installed. Then run the following command, using + the details relevant for your provider / configuration. This will copy all + files from, e.g. `namespace/Kind/name/index.html` to + `namespace/kind/name/index.html`, without removing the original files. + +```sh +techdocs-cli migrate --publisher-type --storage-name --verbose +``` + +3. **Deploy the updated versions of the TechDocs plugins**: Once the migration + above has been run, you can deploy the beta versions of the TechDocs backend + and frontend plugins to your Backstage instance. + +4. **Verify that your TechDocs sites are still loading/accessible**: Try + accessing a TechDocs site using different entity-triplet case variants, e.g. + `/docs/namespace/KIND/name` or `/docs/namespace/kind/name`. Your TechDocs + site should load regardless of the URL path casing you use. + +5. **Clean up the old objects from storage**: Once you've verified that your + TechDocs site is accessible, you can clean up your storage bucket by + re-running the `migrate` command on the TechDocs CLI, but with an additional + `removeOriginal` flag passed: + +```sh +techdocs-cli migrate --publisher-type --storage-name --removeOriginal --verbose +``` + +6. **Update your CI/CD pipelines to use the beta version of the TechDocs CLI**: + Finally, you can update all of your CI/CD pipelines to use at least v0.x.y of + the TechDocs CLI, ensuring that all sites are published to the new, + lower-cased entity triplet paths going forward. + +If you encounter problems running this migration, please [report the +issue][beta-migrate-bug]. You can temporarily revert to pre-beta storage +expectations with a configuration change: + +```yaml +techdocs: + legacyUseCaseSensitiveTripletPaths: true +``` + +[beta-migrate-bug]: +https://github.com/backstage/backstage/issues/new?assignees=&labels=bug&template=bug_template.md&title=[TechDocs]%20Unable%20to%20run%20beta%20migration +[using-cloud-storage]: ./using-cloud-storage.md diff --git a/docs/features/techdocs/using-cloud-storage.md b/docs/features/techdocs/using-cloud-storage.md index 6d221bd9ad..66eaf00663 100644 --- a/docs/features/techdocs/using-cloud-storage.md +++ b/docs/features/techdocs/using-cloud-storage.md @@ -399,9 +399,34 @@ techdocs: Set the configs in your `app-config.yaml` to point to your container name. -https://docs.openstack.org/api-ref/identity/v3/?expanded=password-authentication-with-unscoped-authorization-detail#password-authentication-with-unscoped-authorization +https://docs.openstack.org/api-ref/identity/v3/?expanded=password-authentication-with-unscoped-authorization-detail,authenticating-with-an-application-credential-detail#authenticating-with-an-application-credential for more details. +```yaml +techdocs: + publisher: + type: 'openStackSwift' + openStackSwift: + containerName: 'name-of-techdocs-storage-bucket' + credentials: + id: ${OPENSTACK_SWIFT_STORAGE_APPLICATION_CREDENTIALS_ID} + secret: ${OPENSTACK_SWIFT_STORAGE_APPLICATION_CREDENTIALS_SECRET} + authUrl: ${OPENSTACK_SWIFT_STORAGE_AUTH_URL} + swiftUrl: ${OPENSTACK_SWIFT_STORAGE_SWIFT_URL} +``` + +**4. That's it!** + +Your Backstage app is now ready to use OpenStack Swift Storage for TechDocs, to +store and read the static generated documentation files. When you start the +backend of the app, you should be able to see +`techdocs info Successfully connected to the OpenStack Swift Storage container` +in the logs. + +## Bonus: Migration from old OpenStack Swift Configuration + +Let's assume we have the old OpenStack Swift configuration here. + ```yaml techdocs: publisher: @@ -418,10 +443,40 @@ techdocs: region: ${OPENSTACK_SWIFT_STORAGE_REGION} ``` -**4. That's it!** +##### Step 1: Change the credential keys -Your Backstage app is now ready to use OpenStack Swift Storage for TechDocs, to -store and read the static generated documentation files. When you start the -backend of the app, you should be able to see -`techdocs info Successfully connected to the OpenStack Swift Storage container` -in the logs. +Since the new SDK uses _Application Credentials_ to authenticate OpenStack, we +need to change the keys `credentials.username` to `credentials.id`, +`credentials.password` to `credentials.secret` and use Application Credential ID +and secret here. For more detail about credentials look +[here](https://docs.openstack.org/api-ref/identity/v3/?expanded=password-authentication-with-unscoped-authorization-detail,authenticating-with-an-application-credential-detail#authenticating-with-an-application-credential). + +##### Step 2: Remove the unused keys + +Since the new SDK doesn't use the old way authentication, we don't need the keys +`openStackSwift.keystoneAuthVersion`, `openStackSwift.domainId`, +`openStackSwift.domainName` and `openStackSwift.region`. So you can remove them. + +##### Step 3: Add Swift URL + +The new SDK needs the OpenStack Swift connection URL for connecting the Swift. +So you need to add a new key called `openStackSwift.swiftUrl` and give the +OpenStack Swift url here. Example url should look like that: +`https://example.com:6780/swift/v1` + +##### That's it! + +Your new configuration should look like that! + +```yaml +techdocs: + publisher: + type: 'openStackSwift' + openStackSwift: + containerName: 'name-of-techdocs-storage-bucket' + credentials: + id: ${OPENSTACK_SWIFT_STORAGE_APPLICATION_CREDENTIALS_ID} + secret: ${OPENSTACK_SWIFT_STORAGE_APPLICATION_CREDENTIALS_SECRET} + authUrl: ${OPENSTACK_SWIFT_STORAGE_AUTH_URL} + swiftUrl: ${OPENSTACK_SWIFT_STORAGE_SWIFT_URL} +``` diff --git a/docs/integrations/aws-s3/locations.md b/docs/integrations/aws-s3/locations.md new file mode 100644 index 0000000000..fe1c6050c2 --- /dev/null +++ b/docs/integrations/aws-s3/locations.md @@ -0,0 +1,42 @@ +--- +id: locations +sidebar_label: Locations +title: Amazon Web Services S3 Locations +# prettier-ignore +description: Setting up an integration with Amazon Web Services S3 +--- + +The AWS S3 integration supports loading catalog entities from an S3 Bucket. +Entities can be added to +[static catalog configuration](../../features/software-catalog/configuration.md), +or registered with the +[catalog-import](https://github.com/backstage/backstage/tree/master/plugins/catalog-import) +plugin. + +## Configuration + +To use this integration, add configuration to your `app-config.yaml`: + +```yaml +integrations: + awsS3: + - host: amazonaws.com + accessKeyId: ${AWS_ACCESS_KEY_ID} + secretAccessKey: ${AWS_SECRET_ACCESS_KEY} +``` + +Then make sure the environment variables `AWS_ACCESS_KEY_ID` and +`AWS_SECRET_ACCESS_KEY` are set when you run Backstage. + +Users with multiple AWS accounts may want to use a role for S3 storage that is +in a different AWS account. Using the `roleArn` parameter as seen below, you can +instruct the AWS S3 reader to assume a role before accessing S3: + +```yaml +integrations: + awsS3: + - host: amazonaws.com + accessKeyId: ${AWS_ACCESS_KEY_ID} + secretAccessKey: ${AWS_SECRET_ACCESS_KEY} + roleArn: 'arn:aws:iam::xxxxxxxxxxxx:role/example-role' +``` diff --git a/docs/integrations/gitlab/discovery.md b/docs/integrations/gitlab/discovery.md new file mode 100644 index 0000000000..ade575b74d --- /dev/null +++ b/docs/integrations/gitlab/discovery.md @@ -0,0 +1,36 @@ +--- +id: discovery +title: GitLab Discovery +sidebar_label: Discovery +# prettier-ignore +description: Automatically discovering catalog entities from repositories in GitLab +--- + +The GitLab integration has a special discovery processor for discovering catalog +entities from GitLab. The processor will crawl the GitLab instance and register +entities matching the configured path. This can be useful as an alternative to +static locations or manually adding things to the catalog. + +To use the discovery processor, you'll need a GitLab integration +[set up](locations.md) with a `token`. Then you can add a location target to the +catalog configuration: + +```yaml +catalog: + locations: + - type: gitlab-discovery + target: https://gitlab.com/group/subgroup/blob/main/catalog-info.yaml +``` + +Note the `gitlab-discovery` type, as this is not a regular `url` processor. + +The target is composed of three parts: + +- The base URL, `https://gitlab.com` in this case +- The group path, `group/subgroup` in this case. This is optional: If you omit + this path the processor will scan the entire GitLab instance instead. +- The path within each repository to find the catalog YAML file. This will + usually be `/blob/main/catalog-info.yaml`, `/blob/master/catalog-info.yaml` or + a similar variation for catalog files stored in the root directory of each + repository. If you want to use the repository's default branch use the `*` + wildcard, e.g.: `/blob/*/catalog-info.yaml` diff --git a/microsite/data/plugins/aws-cloudformation.yaml b/microsite/data/plugins/aws-cloudformation.yaml new file mode 100644 index 0000000000..7f0feee143 --- /dev/null +++ b/microsite/data/plugins/aws-cloudformation.yaml @@ -0,0 +1,9 @@ +--- +title: AWS CloudFormation +author: Purple Technology +authorUrl: https://blog.purple-technology.com +category: Infrastructure +description: Load Backstage entities from AWS CloudFormation stacks +documentation: https://github.com/purple-technology/backstage-aws-cloudformation-plugin#readme +iconUrl: https://raw.githubusercontent.com/purple-technology/backstage-aws-cloudformation-plugin/master/docs/cloudformation-logo.png +npmPackageName: 'backstage-aws-cloudformation-plugin' diff --git a/microsite/data/plugins/firehydrant.yaml b/microsite/data/plugins/firehydrant.yaml new file mode 100644 index 0000000000..e4dd5e7416 --- /dev/null +++ b/microsite/data/plugins/firehydrant.yaml @@ -0,0 +1,9 @@ +--- +title: FireHydrant +author: FireHydrant +authorUrl: https://firehydrant.io/ +category: Incident Management +description: View service incidents information from FireHydrant, such as active incidents and incident metrics, directly within Backstage. +documentation: https://github.com/backstage/backstage/blob/master/plugins/firehydrant/README.md +iconUrl: https://raw.githubusercontent.com/backstage/backstage/master/plugins/firehydrant/doc/firehydrant_logo.png +npmPackageName: '@backstage/plugin-firehydrant' diff --git a/microsite/data/plugins/gitlab.yaml b/microsite/data/plugins/gitlab.yaml new file mode 100644 index 0000000000..7a5493c7f4 --- /dev/null +++ b/microsite/data/plugins/gitlab.yaml @@ -0,0 +1,12 @@ +--- +title: GitLab +author: Loblaw +authorUrl: https://github.com/loblaw-sre/backstage-plugin-gitlab +category: CI/CD +description: View GitLab pipelines, merge requests, languages and contributors. +documentation: https://github.com/loblaw-sre/backstage-plugin-gitlab +iconUrl: https://about.gitlab.com/images/press/logo/png/gitlab-icon-rgb.png +npmPackageName: '@loblaw/backstage-plugin-gitlab' +tags: + - ci + - cd diff --git a/microsite/sidebars.json b/microsite/sidebars.json index c22a7e961d..d787fb28aa 100644 --- a/microsite/sidebars.json +++ b/microsite/sidebars.json @@ -104,6 +104,11 @@ ], "Integrations": [ "integrations/index", + { + "type": "subcategory", + "label": "AWS S3", + "ids": ["integrations/aws-s3/locations"] + }, { "type": "subcategory", "label": "Azure", diff --git a/mkdocs.yml b/mkdocs.yml index bd70104abc..73ad52341e 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -79,6 +79,8 @@ nav: - FAQ: 'features/techdocs/FAQ.md' - Integrations: - Overview: 'integrations/index.md' + - AWS S3: + - Locations: 'integrations/aws-s3/locations.md' - Azure: - Locations: 'integrations/azure/locations.md' - Org Data: 'integrations/azure/org.md' diff --git a/packages/app/CHANGELOG.md b/packages/app/CHANGELOG.md index 5ce4f341e1..a3af287fc8 100644 --- a/packages/app/CHANGELOG.md +++ b/packages/app/CHANGELOG.md @@ -1,5 +1,37 @@ # example-app +## 0.2.43 + +### Patch Changes + +- Updated dependencies + - @backstage/plugin-explore@0.3.14 + - @backstage/core-app-api@0.1.10 + - @backstage/plugin-catalog-import@0.5.19 + - @backstage/plugin-catalog@0.6.13 + - @backstage/plugin-org@0.3.20 + - @backstage/plugin-catalog-react@0.4.4 + - @backstage/core-components@0.3.3 + - @backstage/plugin-techdocs@0.11.0 + - @backstage/plugin-search@0.4.8 + +## 0.2.42 + +### Patch Changes + +- Updated dependencies + - @backstage/plugin-scaffolder@0.11.1 + - @backstage/plugin-home@0.2.0 + - @backstage/cli@0.7.9 + - @backstage/core-app-api@0.1.9 + - @backstage/plugin-catalog-import@0.5.18 + - @backstage/plugin-catalog@0.6.12 + - @backstage/plugin-catalog-react@0.4.3 + - @backstage/plugin-sentry@0.3.19 + - @backstage/plugin-jenkins@0.5.4 + - @backstage/core-components@0.3.2 + - @backstage/theme@0.2.10 + ## 0.2.41 ### Patch Changes diff --git a/packages/app/package.json b/packages/app/package.json index dcf14c88c1..162474fd0e 100644 --- a/packages/app/package.json +++ b/packages/app/package.json @@ -1,54 +1,55 @@ { "name": "example-app", - "version": "0.2.41", + "version": "0.2.43", "private": true, "bundled": true, "dependencies": { "@backstage/catalog-model": "^0.9.0", - "@backstage/cli": "^0.7.8", - "@backstage/core-app-api": "^0.1.8", - "@backstage/core-components": "^0.3.1", + "@backstage/cli": "^0.7.9", + "@backstage/core-app-api": "^0.1.10", + "@backstage/core-components": "^0.3.3", "@backstage/core-plugin-api": "^0.1.6", "@backstage/integration-react": "^0.1.7", "@backstage/plugin-api-docs": "^0.6.6", "@backstage/plugin-badges": "^0.2.7", - "@backstage/plugin-catalog": "^0.6.11", - "@backstage/plugin-catalog-import": "^0.5.17", - "@backstage/plugin-catalog-react": "^0.4.2", + "@backstage/plugin-catalog": "^0.6.13", + "@backstage/plugin-catalog-import": "^0.5.19", + "@backstage/plugin-catalog-react": "^0.4.4", "@backstage/plugin-circleci": "^0.2.21", "@backstage/plugin-cloudbuild": "^0.2.21", "@backstage/plugin-code-coverage": "^0.1.9", "@backstage/plugin-cost-insights": "^0.11.4", - "@backstage/plugin-explore": "^0.3.13", + "@backstage/plugin-explore": "^0.3.14", "@backstage/plugin-gcp-projects": "^0.3.2", "@backstage/plugin-github-actions": "^0.4.16", "@backstage/plugin-graphiql": "^0.2.14", - "@backstage/plugin-jenkins": "^0.5.3", + "@backstage/plugin-home": "^0.2.0", + "@backstage/plugin-jenkins": "^0.5.4", "@backstage/plugin-kafka": "^0.2.13", "@backstage/plugin-kubernetes": "^0.4.11", "@backstage/plugin-lighthouse": "^0.2.23", "@backstage/plugin-newrelic": "^0.3.2", - "@backstage/plugin-org": "^0.3.19", + "@backstage/plugin-org": "^0.3.20", "@backstage/plugin-pagerduty": "0.3.11", "@backstage/plugin-rollbar": "^0.3.12", - "@backstage/plugin-scaffolder": "^0.11.0", - "@backstage/plugin-search": "^0.4.7", - "@backstage/plugin-sentry": "^0.3.18", + "@backstage/plugin-scaffolder": "^0.11.1", + "@backstage/plugin-search": "^0.4.8", + "@backstage/plugin-sentry": "^0.3.19", "@backstage/plugin-shortcuts": "^0.1.6", "@backstage/plugin-tech-radar": "^0.4.5", - "@backstage/plugin-techdocs": "^0.10.4", + "@backstage/plugin-techdocs": "^0.11.0", "@backstage/plugin-todo": "^0.1.7", "@backstage/plugin-user-settings": "^0.3.3", "@backstage/search-common": "^0.1.3", - "@backstage/theme": "^0.2.9", + "@backstage/theme": "^0.2.10", "@material-ui/core": "^4.12.2", "@material-ui/icons": "^4.9.1", "@material-ui/lab": "4.0.0-alpha.45", "@octokit/rest": "^18.5.3", - "@roadiehq/backstage-plugin-buildkite": "^1.0.6", - "@roadiehq/backstage-plugin-github-insights": "^1.1.20", - "@roadiehq/backstage-plugin-github-pull-requests": "^1.0.10", - "@roadiehq/backstage-plugin-travis-ci": "^1.0.8", + "@roadiehq/backstage-plugin-buildkite": "^1.0.8", + "@roadiehq/backstage-plugin-github-insights": "^1.1.23", + "@roadiehq/backstage-plugin-github-pull-requests": "^1.0.13", + "@roadiehq/backstage-plugin-travis-ci": "^1.0.11", "history": "^5.0.0", "prop-types": "^15.7.2", "react": "^16.12.0", diff --git a/packages/app/src/App.tsx b/packages/app/src/App.tsx index 1132368a91..81dbdf4cac 100644 --- a/packages/app/src/App.tsx +++ b/packages/app/src/App.tsx @@ -20,6 +20,7 @@ import { OAuthRequestDialog, SignInPage, } from '@backstage/core-components'; +import { HomepageCompositionRoot } from '@backstage/plugin-home'; import { apiDocsPlugin, ApiExplorerPage } from '@backstage/plugin-api-docs'; import { CatalogEntityPage, @@ -45,16 +46,13 @@ import { ScaffolderPage, scaffolderPlugin, ScaffolderFieldExtensions, - RepoUrlPickerFieldExtension, - OwnerPickerFieldExtension, - EntityPickerFieldExtension, - EntityNamePickerFieldExtension, } from '@backstage/plugin-scaffolder'; import { SearchPage } from '@backstage/plugin-search'; import { TechRadarPage } from '@backstage/plugin-tech-radar'; import { DefaultTechDocsHome, TechDocsIndexPage, + techdocsPlugin, TechDocsReaderPage, } from '@backstage/plugin-techdocs'; import { UserSettingsPage } from '@backstage/plugin-user-settings'; @@ -67,6 +65,8 @@ import { Root } from './components/Root'; import { entityPage } from './components/catalog/EntityPage'; import { searchPage } from './components/search/SearchPage'; import { LowerCaseValuePickerFieldExtension } from './components/scaffolder/customScaffolderExtensions'; +import { HomePage } from './components/home/HomePage'; + import { providers } from './identityProviders'; import * as plugins from './plugins'; @@ -93,6 +93,7 @@ const app = createApp({ bindRoutes({ bind }) { bind(catalogPlugin.externalRoutes, { createComponent: scaffolderPlugin.routes.root, + viewTechDoc: techdocsPlugin.routes.docRoot, }); bind(apiDocsPlugin.externalRoutes, { createComponent: scaffolderPlugin.routes.root, @@ -112,6 +113,10 @@ const AppRouter = app.getRouter(); const routes = ( + {/* TODO(rubenl): Move this to / once its more mature and components exist */} + }> + + } /> }> - - - - diff --git a/packages/app/src/components/catalog/EntityPage.tsx b/packages/app/src/components/catalog/EntityPage.tsx index d84a7341bc..4a392e0f05 100644 --- a/packages/app/src/components/catalog/EntityPage.tsx +++ b/packages/app/src/components/catalog/EntityPage.tsx @@ -211,20 +211,8 @@ const cicdCard = ( ); -const errorsContent = ( - - - - - - - - - -); - -const overviewContent = ( - +const entityWarningContent = ( + <> @@ -240,7 +228,24 @@ const overviewContent = ( + +); +const errorsContent = ( + + + + + + + + + +); + +const overviewContent = ( + + {entityWarningContent} @@ -448,6 +453,7 @@ const apiPage = ( + {entityWarningContent} @@ -478,6 +484,7 @@ const userPage = ( + {entityWarningContent} @@ -493,6 +500,7 @@ const groupPage = ( + {entityWarningContent} @@ -511,6 +519,7 @@ const systemPage = ( + {entityWarningContent} @@ -535,6 +544,7 @@ const domainPage = ( + {entityWarningContent} diff --git a/packages/app/src/components/home/HomePage.tsx b/packages/app/src/components/home/HomePage.tsx new file mode 100644 index 0000000000..e171e22ae9 --- /dev/null +++ b/packages/app/src/components/home/HomePage.tsx @@ -0,0 +1,72 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import React from 'react'; +import Grid from '@material-ui/core/Grid'; +import { + RandomJokeHomePageComponent, + ComponentAccordion, + ComponentTabs, + ComponentTab, +} from '@backstage/plugin-home'; + +export const HomePage = () => ( + + + + + + + + + + + ( + + ), + }, + { + label: 'Any', + Component: () => ( + + ), + }, + ]} + /> + + +); diff --git a/packages/backend-common/CHANGELOG.md b/packages/backend-common/CHANGELOG.md index 297f07b0b4..f3fcd5a855 100644 --- a/packages/backend-common/CHANGELOG.md +++ b/packages/backend-common/CHANGELOG.md @@ -1,5 +1,30 @@ # @backstage/backend-common +## 0.9.0 + +### Minor Changes + +- a365f1faf: The `ZipArchiveResponse` class now accepts an optional `stripFirstDirectory` parameter. Note that its default value is `false`, which leads to a breaking change in behaviour to previous versions of the class. If you use this class explicitly and want to retain the old behaviour, add a `true` parameter value to its constructor. + +### Patch Changes + +- Updated dependencies + - @backstage/integration@0.6.2 + - @backstage/config@0.1.8 + +## 0.8.10 + +### Patch Changes + +- 8543d9890: Add an optional `info` parameter to the `readTree` filter option with a `size` property. +- 4d909268c: Read responses in `UrlReader#read()` as array buffer instead of as text to allow reading non-text locations such as images. +- 9b4604b38: Add support for watching configuration by implementing the `subscribe` method in the configuration returned by `loadBackendConfig`. +- b8cb12009: Add AWS S3 URL Reader +- Updated dependencies + - @backstage/config@0.1.7 + - @backstage/config-loader@0.6.7 + - @backstage/integration@0.6.1 + ## 0.8.9 ### Patch Changes diff --git a/packages/backend-common/api-report.md b/packages/backend-common/api-report.md index 2652dad122..de8c1d4101 100644 --- a/packages/backend-common/api-report.md +++ b/packages/backend-common/api-report.md @@ -42,23 +42,14 @@ export class AzureUrlReader implements UrlReader { treeResponseFactory: ReadTreeResponseFactory; }, ); - // Warning: (ae-forgotten-export) The symbol "ReaderFactory" needs to be exported by the entry point index.d.ts - // // (undocumented) static factory: ReaderFactory; // (undocumented) read(url: string): Promise; - // Warning: (ae-forgotten-export) The symbol "ReadTreeOptions" needs to be exported by the entry point index.d.ts - // // (undocumented) readTree(url: string, options?: ReadTreeOptions): Promise; - // Warning: (ae-forgotten-export) The symbol "ReadUrlOptions" needs to be exported by the entry point index.d.ts - // Warning: (ae-forgotten-export) The symbol "ReadUrlResponse" needs to be exported by the entry point index.d.ts - // // (undocumented) readUrl(url: string, _options?: ReadUrlOptions): Promise; - // Warning: (ae-forgotten-export) The symbol "SearchOptions" needs to be exported by the entry point index.d.ts - // // (undocumented) search(url: string, options?: SearchOptions): Promise; // (undocumented) @@ -415,6 +406,28 @@ export type PluginEndpointDiscovery = { getExternalBaseUrl(pluginId: string): Promise; }; +// Warning: (ae-missing-release-tag) "ReaderFactory" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public +export type ReaderFactory = (options: { + config: Config; + logger: Logger_2; + treeResponseFactory: ReadTreeResponseFactory; +}) => UrlReaderPredicateTuple[]; + +// Warning: (ae-missing-release-tag) "ReadTreeOptions" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public +export type ReadTreeOptions = { + filter?( + path: string, + info?: { + size: number; + }, + ): boolean; + etag?: string; +}; + // Warning: (ae-missing-release-tag) "ReadTreeResponse" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public @@ -425,6 +438,18 @@ export type ReadTreeResponse = { etag: string; }; +// Warning: (ae-missing-release-tag) "ReadTreeResponseFactory" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export interface ReadTreeResponseFactory { + // Warning: (ae-forgotten-export) The symbol "FromArchiveOptions" needs to be exported by the entry point index.d.ts + // + // (undocumented) + fromTarArchive(options: FromArchiveOptions): Promise; + // (undocumented) + fromZipArchive(options: FromArchiveOptions): Promise; +} + // Warning: (ae-missing-release-tag) "ReadTreeResponseFile" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public @@ -433,6 +458,21 @@ export type ReadTreeResponseFile = { content(): Promise; }; +// Warning: (ae-missing-release-tag) "ReadUrlOptions" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public +export type ReadUrlOptions = { + etag?: string; +}; + +// Warning: (ae-missing-release-tag) "ReadUrlResponse" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public +export type ReadUrlResponse = { + buffer(): Promise; + etag?: string; +}; + // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen // Warning: (ae-missing-release-tag) "requestLoggingHandler" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // @@ -472,6 +512,13 @@ export type RunContainerOptions = { pullImage?: boolean; }; +// Warning: (ae-missing-release-tag) "SearchOptions" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public +export type SearchOptions = { + etag?: string; +}; + // Warning: (ae-missing-release-tag) "SearchResponse" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public @@ -561,6 +608,14 @@ export type UrlReader = { search(url: string, options?: SearchOptions): Promise; }; +// Warning: (ae-missing-release-tag) "UrlReaderPredicateTuple" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export type UrlReaderPredicateTuple = { + predicate: (url: URL) => boolean; + reader: UrlReader; +}; + // Warning: (ae-missing-release-tag) "UrlReaders" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public @@ -592,8 +647,7 @@ export function useHotMemoize(_module: NodeModule, valueFactory: () => T): T; // // src/cache/types.d.ts:34:5 - (ae-forgotten-export) The symbol "ClientOptions" needs to be exported by the entry point index.d.ts // src/middleware/errorHandler.d.ts:17:26 - (tsdoc-malformed-html-name) Invalid HTML element: A space is not allowed here -// src/reading/AzureUrlReader.d.ts:9:9 - (ae-forgotten-export) The symbol "ReadTreeResponseFactory" needs to be exported by the entry point index.d.ts -// src/reading/types.d.ts:106:5 - (ae-forgotten-export) The symbol "ReadTreeResponseDirOptions" needs to be exported by the entry point index.d.ts +// src/reading/types.d.ts:108:5 - (ae-forgotten-export) The symbol "ReadTreeResponseDirOptions" needs to be exported by the entry point index.d.ts // src/service/types.d.ts:12:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen // src/service/types.d.ts:22:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen // src/service/types.d.ts:30:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen diff --git a/packages/backend-common/package.json b/packages/backend-common/package.json index 4c7aacdf96..15a559f960 100644 --- a/packages/backend-common/package.json +++ b/packages/backend-common/package.json @@ -1,7 +1,7 @@ { "name": "@backstage/backend-common", "description": "Common functionality library for Backstage backends", - "version": "0.8.9", + "version": "0.9.0", "main": "src/index.ts", "types": "src/index.ts", "private": false, @@ -30,16 +30,17 @@ }, "dependencies": { "@backstage/cli-common": "^0.1.2", - "@backstage/config": "^0.1.6", - "@backstage/config-loader": "^0.6.6", + "@backstage/config": "^0.1.8", + "@backstage/config-loader": "^0.6.7", "@backstage/errors": "^0.1.1", - "@backstage/integration": "^0.6.0", + "@backstage/integration": "^0.6.2", "@google-cloud/storage": "^5.8.0", "@octokit/rest": "^18.5.3", "@types/cors": "^2.8.6", "@types/dockerode": "^3.2.1", "@types/express": "^4.17.6", "archiver": "^5.0.2", + "aws-sdk": "^2.840.0", "compression": "^1.7.4", "concat-stream": "^2.0.0", "cors": "^2.8.5", @@ -76,7 +77,7 @@ } }, "devDependencies": { - "@backstage/cli": "^0.7.8", + "@backstage/cli": "^0.7.9", "@backstage/test-utils": "^0.1.17", "@types/archiver": "^5.1.0", "@types/compression": "^1.7.0", @@ -92,6 +93,7 @@ "@types/tar": "^4.0.3", "@types/unzipper": "^0.10.3", "@types/webpack-env": "^1.15.2", + "aws-sdk-mock": "^5.2.1", "get-port": "^5.1.1", "http-errors": "^1.7.3", "jest": "^26.0.1", diff --git a/packages/backend-common/src/config.ts b/packages/backend-common/src/config.ts index 1316642ae1..7c660ee4e6 100644 --- a/packages/backend-common/src/config.ts +++ b/packages/backend-common/src/config.ts @@ -18,32 +18,142 @@ import { resolve as resolvePath } from 'path'; import parseArgs from 'minimist'; import { Logger } from 'winston'; import { findPaths } from '@backstage/cli-common'; -import { Config, ConfigReader } from '@backstage/config'; +import { Config, ConfigReader, JsonValue } from '@backstage/config'; import { loadConfig } from '@backstage/config-loader'; +class ObservableConfigProxy implements Config { + private config: Config = new ConfigReader({}); + + private readonly subscribers: (() => void)[] = []; + + constructor(private readonly logger: Logger) {} + + setConfig(config: Config) { + this.config = config; + for (const subscriber of this.subscribers) { + try { + subscriber(); + } catch (error) { + this.logger.error(`Config subscriber threw error, ${error}`); + } + } + } + + subscribe(onChange: () => void): { unsubscribe: () => void } { + this.subscribers.push(onChange); + return { + unsubscribe: () => { + const index = this.subscribers.indexOf(onChange); + if (index >= 0) { + this.subscribers.splice(index, 1); + } + }, + }; + } + + has(key: string): boolean { + return this.config.has(key); + } + keys(): string[] { + return this.config.keys(); + } + get(key?: string): T { + return this.config.get(key); + } + getOptional(key?: string): T | undefined { + return this.config.getOptional(key); + } + getConfig(key: string): Config { + return this.config.getConfig(key); + } + getOptionalConfig(key: string): Config | undefined { + return this.config.getOptionalConfig(key); + } + getConfigArray(key: string): Config[] { + return this.config.getConfigArray(key); + } + getOptionalConfigArray(key: string): Config[] | undefined { + return this.config.getOptionalConfigArray(key); + } + getNumber(key: string): number { + return this.config.getNumber(key); + } + getOptionalNumber(key: string): number | undefined { + return this.config.getOptionalNumber(key); + } + getBoolean(key: string): boolean { + return this.config.getBoolean(key); + } + getOptionalBoolean(key: string): boolean | undefined { + return this.config.getOptionalBoolean(key); + } + getString(key: string): string { + return this.config.getString(key); + } + getOptionalString(key: string): string | undefined { + return this.config.getOptionalString(key); + } + getStringArray(key: string): string[] { + return this.config.getStringArray(key); + } + getOptionalStringArray(key: string): string[] | undefined { + return this.config.getOptionalStringArray(key); + } +} + type Options = { logger: Logger; // process.argv or any other overrides argv: string[]; }; +// A global used to ensure that only a single file watcher is active at a time. +let currentCancelFunc: () => void; + /** - * Load configuration for a Backend + * Load configuration for a Backend. + * + * This function should only be called once, during the initialization of the backend. */ export async function loadBackendConfig(options: Options): Promise { const args = parseArgs(options.argv); - const configOpts: string[] = [args.config ?? []].flat(); + const configPaths: string[] = [args.config ?? []].flat(); + + const config = new ObservableConfigProxy(options.logger); /* eslint-disable-next-line no-restricted-syntax */ const paths = findPaths(__dirname); + const configs = await loadConfig({ configRoot: paths.targetRoot, - configPaths: configOpts.map(opt => resolvePath(opt)), + configPaths: configPaths.map(opt => resolvePath(opt)), + watch: { + onChange(newConfigs) { + options.logger.info( + `Reloaded config from ${newConfigs.map(c => c.context).join(', ')}`, + ); + + config.setConfig(ConfigReader.fromConfigs(newConfigs)); + }, + stopSignal: new Promise(resolve => { + if (currentCancelFunc) { + currentCancelFunc(); + } + currentCancelFunc = resolve; + + // For reloads of this module we need to use a dispose handler rather than the global. + if (module.hot) { + module.hot.addDisposeHandler(resolve); + } + }), + }, }); options.logger.info( `Loaded config from ${configs.map(c => c.context).join(', ')}`, ); - return ConfigReader.fromConfigs(configs); + config.setConfig(ConfigReader.fromConfigs(configs)); + + return config; } diff --git a/packages/backend-common/src/reading/AwsS3UrlReader.test.ts b/packages/backend-common/src/reading/AwsS3UrlReader.test.ts new file mode 100644 index 0000000000..084f7e3e13 --- /dev/null +++ b/packages/backend-common/src/reading/AwsS3UrlReader.test.ts @@ -0,0 +1,232 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { ConfigReader, JsonObject } from '@backstage/config'; +import { getVoidLogger } from '../logging'; +import { DefaultReadTreeResponseFactory } from './tree'; +import { AwsS3UrlReader } from './AwsS3UrlReader'; +import { + AwsS3Integration, + readAwsS3IntegrationConfig, +} from '@backstage/integration'; +import { UrlReaderPredicateTuple } from './types'; +import AWSMock from 'aws-sdk-mock'; +import aws from 'aws-sdk'; +import path from 'path'; + +describe('AwsS3UrlReader', () => { + const createReader = (config: JsonObject): UrlReaderPredicateTuple[] => { + return AwsS3UrlReader.factory({ + config: new ConfigReader(config), + logger: getVoidLogger(), + treeResponseFactory: DefaultReadTreeResponseFactory.create({ + config: new ConfigReader({}), + }), + }); + }; + + afterEach(() => { + AWSMock.restore(); + }); + + it('creates a dummy reader without the awsS3 field', () => { + const entries = createReader({ + integrations: {}, + }); + + expect(entries).toHaveLength(1); + }); + + it('creates a reader with credentials correctly configured', () => { + const awsS3Integrations = []; + awsS3Integrations.push({ + host: 'amazonaws.com', + accessKeyId: 'fakekey', + secretAccessKey: 'fakekey', + }); + + const entries = createReader({ + integrations: { + awsS3: awsS3Integrations, + }, + }); + + expect(entries).toHaveLength(1); + }); + + it('creates a reader with default credentials provider', () => { + const awsS3Integrations = []; + awsS3Integrations.push({ + host: 'amazonaws.com', + }); + + const entries = createReader({ + integrations: { + awsS3: awsS3Integrations, + }, + }); + + expect(entries).toHaveLength(1); + }); + + describe('predicates', () => { + const readers = createReader({ + integrations: { + awsS3: [{}], + }, + }); + const predicate = readers[0].predicate; + + it('returns true for the correct aws s3 storage host', () => { + expect( + predicate(new URL('https://test-bucket.s3.us-east-2.amazonaws.com')), + ).toBe(true); + }); + + it('returns true for a url with the full path and the correct host', () => { + expect( + predicate( + new URL( + 'https://test-bucket.s3.us-east-2.amazonaws.com/team/service/catalog-info.yaml', + ), + ), + ).toBe(true); + }); + + it('returns false for an incorrect host', () => { + expect(predicate(new URL('https://amazon.com'))).toBe(false); + }); + + it('returns false for a completely different host', () => { + expect(predicate(new URL('https://storage.cloud.google.com'))).toBe( + false, + ); + }); + + it("returns true for a url with a bucket with '.'", () => { + expect( + predicate( + new URL( + 'https://test.bucket.s3.us-east-2.amazonaws.com/team/service/catalog-info.yaml', + ), + ), + ).toBe(true); + }); + }); + + describe('read', () => { + AWSMock.setSDKInstance(aws); + AWSMock.mock( + 'S3', + 'getObject', + Buffer.from( + require('fs').readFileSync( + path.resolve( + 'src', + 'reading', + '__fixtures__', + 'awsS3-mock-object.yaml', + ), + ), + ), + ); + const s3 = new aws.S3(); + const awsS3UrlReader = new AwsS3UrlReader( + new AwsS3Integration( + readAwsS3IntegrationConfig( + new ConfigReader({ + host: 'amazonaws.com', + accessKeyId: 'fake-access-key', + secretAccessKey: 'fake-secret-key', + }), + ), + ), + s3, + ); + + it('returns contents of an object in a bucket', async () => { + const response = await awsS3UrlReader.read( + 'https://test-bucket.s3.us-east-2.amazonaws.com/awsS3-mock-object.yaml', + ); + expect(response.toString().trim()).toBe('site_name: Test'); + }); + + it('rejects unknown targets', async () => { + await expect( + awsS3UrlReader.read( + 'https://test-bucket.s3.us-east-2.NOTamazonaws.com/file.yaml', + ), + ).rejects.toThrow( + Error( + `Could not retrieve file from S3: not a valid AWS S3 URL: https://test-bucket.s3.us-east-2.NOTamazonaws.com/file.yaml`, + ), + ); + }); + }); + + describe('readUrl', () => { + AWSMock.setSDKInstance(aws); + + AWSMock.mock( + 'S3', + 'getObject', + Buffer.from( + require('fs').readFileSync( + path.resolve( + 'src', + 'reading', + '__fixtures__', + 'awsS3-mock-object.yaml', + ), + ), + ), + ); + + const s3 = new aws.S3(); + + const awsS3UrlReader = new AwsS3UrlReader( + new AwsS3Integration( + readAwsS3IntegrationConfig( + new ConfigReader({ + host: 'amazonaws.com', + accessKeyId: 'fake-access-key', + secretAccessKey: 'fake-secret-key', + }), + ), + ), + s3, + ); + + it('returns contents of an object in a bucket', async () => { + const response = await awsS3UrlReader.readUrl( + 'https://test-bucket.s3.us-east-2.amazonaws.com/awsS3-mock-object.yaml', + ); + const buffer = await response.buffer(); + expect(buffer.toString().trim()).toBe('site_name: Test'); + }); + + it('rejects unknown targets', async () => { + await expect( + awsS3UrlReader.readUrl( + 'https://test-bucket.s3.us-east-2.NOTamazonaws.com/file.yaml', + ), + ).rejects.toThrow( + Error( + `Could not retrieve file from S3: not a valid AWS S3 URL: https://test-bucket.s3.us-east-2.NOTamazonaws.com/file.yaml`, + ), + ); + }); + }); +}); diff --git a/packages/backend-common/src/reading/AwsS3UrlReader.ts b/packages/backend-common/src/reading/AwsS3UrlReader.ts new file mode 100644 index 0000000000..d119115930 --- /dev/null +++ b/packages/backend-common/src/reading/AwsS3UrlReader.ts @@ -0,0 +1,175 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import aws, { Credentials, S3 } from 'aws-sdk'; +import { CredentialsOptions } from 'aws-sdk/lib/credentials'; +import { + ReaderFactory, + ReadTreeResponse, + ReadUrlOptions, + ReadUrlResponse, + SearchResponse, + UrlReader, +} from './types'; +import getRawBody from 'raw-body'; +import { AwsS3Integration, ScmIntegrations } from '@backstage/integration'; + +const parseURL = ( + url: string, +): { path: string; bucket: string; region: string } => { + let { host, pathname } = new URL(url); + + /** + * Removes the leading '/' from the pathname to be processed + * as a parameter by AWS S3 SDK getObject method. + */ + pathname = pathname.substr(1); + + /** + * Checks that the given URL is a valid S3 object url. + * Format of a Valid S3 URL: https://bucket-name.s3.Region.amazonaws.com/keyname + */ + const validHost = new RegExp( + /^[a-z\d][a-z\d\.-]{1,61}[a-z\d]\.s3\.[a-z\d-]+\.amazonaws.com$/, + ); + if (!validHost.test(host)) { + throw new Error(`not a valid AWS S3 URL: ${url}`); + } + + const [bucket] = host.split(/\.s3\.[a-z\d-]+\.amazonaws.com/); + host = host.substring(bucket.length); + const [, , region, ,] = host.split('.'); + + return { + path: pathname, + bucket: bucket, + region: region, + }; +}; + +export class AwsS3UrlReader implements UrlReader { + static factory: ReaderFactory = ({ config }) => { + const integrations = ScmIntegrations.fromConfig(config); + + return integrations.awsS3.list().map(integration => { + const creds = AwsS3UrlReader.buildCredentials(integration); + const s3 = new S3({ + apiVersion: '2006-03-01', + credentials: creds, + }); + const reader = new AwsS3UrlReader(integration, s3); + const predicate = (url: URL) => + url.host.endsWith(integration.config.host); + return { reader, predicate }; + }); + }; + + constructor( + private readonly integration: AwsS3Integration, + private readonly s3: S3, + ) {} + + /** + * If accesKeyId and secretAccessKey are missing, the standard credentials provider chain will be used: + * https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/auth/DefaultAWSCredentialsProviderChain.html + */ + private static buildCredentials( + integration?: AwsS3Integration, + ): Credentials | CredentialsOptions | undefined { + if (!integration) { + return undefined; + } + + const accessKeyId = integration.config.accessKeyId; + const secretAccessKey = integration.config.secretAccessKey; + let explicitCredentials: Credentials | undefined; + + if (accessKeyId && secretAccessKey) { + explicitCredentials = new Credentials({ + accessKeyId, + secretAccessKey, + }); + } + + const roleArn = integration.config.roleArn; + if (roleArn) { + return new aws.ChainableTemporaryCredentials({ + masterCredentials: explicitCredentials, + params: { + RoleSessionName: 'backstage-aws-s3-url-reader', + RoleArn: roleArn, + }, + }); + } + + return explicitCredentials; + } + + async read(url: string): Promise { + const response = await this.readUrl(url); + return response.buffer(); + } + + async readUrl( + url: string, + options?: ReadUrlOptions, + ): Promise { + try { + const { path, bucket, region } = parseURL(url); + aws.config.update({ region: region }); + + let params; + if (options?.etag) { + params = { + Bucket: bucket, + Key: path, + IfNoneMatch: options.etag, + }; + } else { + params = { + Bucket: bucket, + Key: path, + }; + } + + const response = this.s3.getObject(params); + const buffer = await getRawBody(response.createReadStream()); + const etag = (await response.promise()).ETag; + + return { + buffer: async () => buffer, + etag: etag, + }; + } catch (e) { + throw new Error(`Could not retrieve file from S3: ${e.message}`); + } + } + + async readTree(): Promise { + throw new Error('AwsS3Reader does not implement readTree'); + } + + async search(): Promise { + throw new Error('AwsS3Reader does not implement search'); + } + + toString() { + const secretAccessKey = this.integration.config.secretAccessKey; + return `awsS3{host=${this.integration.config.host},authed=${Boolean( + secretAccessKey, + )}}`; + } +} diff --git a/packages/backend-common/src/reading/AzureUrlReader.ts b/packages/backend-common/src/reading/AzureUrlReader.ts index d2c056508d..7b4b1e345e 100644 --- a/packages/backend-common/src/reading/AzureUrlReader.ts +++ b/packages/backend-common/src/reading/AzureUrlReader.ts @@ -70,7 +70,7 @@ export class AzureUrlReader implements UrlReader { // for private repos when PAT is not valid, Azure API returns a http status code 203 with sign in page html if (response.ok && response.status !== 203) { - return Buffer.from(await response.text()); + return Buffer.from(await response.arrayBuffer()); } const message = `${url} could not be read as ${builtUrl}, ${response.status} ${response.statusText}`; diff --git a/packages/backend-common/src/reading/BitbucketUrlReader.ts b/packages/backend-common/src/reading/BitbucketUrlReader.ts index 2a979a464f..7925212b20 100644 --- a/packages/backend-common/src/reading/BitbucketUrlReader.ts +++ b/packages/backend-common/src/reading/BitbucketUrlReader.ts @@ -86,7 +86,7 @@ export class BitbucketUrlReader implements UrlReader { } if (response.ok) { - return Buffer.from(await response.text()); + return Buffer.from(await response.arrayBuffer()); } const message = `${url} could not be read as ${bitbucketUrl}, ${response.status} ${response.statusText}`; diff --git a/packages/backend-common/src/reading/FetchUrlReader.ts b/packages/backend-common/src/reading/FetchUrlReader.ts index 3177ee1f8e..6b45d99cc3 100644 --- a/packages/backend-common/src/reading/FetchUrlReader.ts +++ b/packages/backend-common/src/reading/FetchUrlReader.ts @@ -82,7 +82,7 @@ export class FetchUrlReader implements UrlReader { if (response.ok) { return { - buffer: async () => Buffer.from(await response.text()), + buffer: async () => Buffer.from(await response.arrayBuffer()), etag: response.headers.get('ETag') ?? undefined, }; } diff --git a/packages/backend-common/src/reading/GithubUrlReader.ts b/packages/backend-common/src/reading/GithubUrlReader.ts index 40d36fb2a0..5dc1430ea8 100644 --- a/packages/backend-common/src/reading/GithubUrlReader.ts +++ b/packages/backend-common/src/reading/GithubUrlReader.ts @@ -119,7 +119,7 @@ export class GithubUrlReader implements UrlReader { if (response.ok) { return { - buffer: async () => Buffer.from(await response.text()), + buffer: async () => Buffer.from(await response.arrayBuffer()), etag: response.headers.get('ETag') ?? undefined, }; } diff --git a/packages/backend-common/src/reading/GitlabUrlReader.ts b/packages/backend-common/src/reading/GitlabUrlReader.ts index 27c31c7673..baee64fab2 100644 --- a/packages/backend-common/src/reading/GitlabUrlReader.ts +++ b/packages/backend-common/src/reading/GitlabUrlReader.ts @@ -84,7 +84,7 @@ export class GitlabUrlReader implements UrlReader { if (response.ok) { return { - buffer: async () => Buffer.from(await response.text()), + buffer: async () => Buffer.from(await response.arrayBuffer()), etag: response.headers.get('ETag') ?? undefined, }; } diff --git a/packages/backend-common/src/reading/UrlReaders.ts b/packages/backend-common/src/reading/UrlReaders.ts index 2b3a2f166c..f9a4865717 100644 --- a/packages/backend-common/src/reading/UrlReaders.ts +++ b/packages/backend-common/src/reading/UrlReaders.ts @@ -25,6 +25,7 @@ import { GitlabUrlReader } from './GitlabUrlReader'; import { DefaultReadTreeResponseFactory } from './tree'; import { FetchUrlReader } from './FetchUrlReader'; import { GoogleGcsUrlReader } from './GoogleGcsUrlReader'; +import { AwsS3UrlReader } from './AwsS3UrlReader'; type CreateOptions = { /** Root config object */ @@ -74,6 +75,7 @@ export class UrlReaders { GithubUrlReader.factory, GitlabUrlReader.factory, GoogleGcsUrlReader.factory, + AwsS3UrlReader.factory, FetchUrlReader.factory, ]), }); diff --git a/packages/backend-common/src/reading/__fixtures__/awsS3-mock-object.yaml b/packages/backend-common/src/reading/__fixtures__/awsS3-mock-object.yaml new file mode 100644 index 0000000000..7470c0e8a3 --- /dev/null +++ b/packages/backend-common/src/reading/__fixtures__/awsS3-mock-object.yaml @@ -0,0 +1 @@ +site_name: Test diff --git a/packages/backend-common/src/reading/__fixtures__/mock-main.zip b/packages/backend-common/src/reading/__fixtures__/mock-main.zip index beee59d3a0..ae362448fc 100644 Binary files a/packages/backend-common/src/reading/__fixtures__/mock-main.zip and b/packages/backend-common/src/reading/__fixtures__/mock-main.zip differ diff --git a/packages/backend-common/src/reading/__fixtures__/mock-with-extra-root-dir.zip b/packages/backend-common/src/reading/__fixtures__/mock-with-extra-root-dir.zip new file mode 100644 index 0000000000..beee59d3a0 Binary files /dev/null and b/packages/backend-common/src/reading/__fixtures__/mock-with-extra-root-dir.zip differ diff --git a/packages/backend-common/src/reading/index.ts b/packages/backend-common/src/reading/index.ts index 4c601556d7..207f01b37f 100644 --- a/packages/backend-common/src/reading/index.ts +++ b/packages/backend-common/src/reading/index.ts @@ -19,10 +19,17 @@ export { BitbucketUrlReader } from './BitbucketUrlReader'; export { GithubUrlReader } from './GithubUrlReader'; export { GitlabUrlReader } from './GitlabUrlReader'; export type { + ReaderFactory, + ReadTreeOptions, ReadTreeResponse, + ReadTreeResponseFactory, ReadTreeResponseFile, + ReadUrlOptions, + ReadUrlResponse, + SearchOptions, SearchResponse, SearchResponseFile, UrlReader, + UrlReaderPredicateTuple, } from './types'; export { UrlReaders } from './UrlReaders'; diff --git a/packages/backend-common/src/reading/tree/TarArchiveResponse.ts b/packages/backend-common/src/reading/tree/TarArchiveResponse.ts index 173f6a886a..f7ab7f5349 100644 --- a/packages/backend-common/src/reading/tree/TarArchiveResponse.ts +++ b/packages/backend-common/src/reading/tree/TarArchiveResponse.ts @@ -43,7 +43,7 @@ export class TarArchiveResponse implements ReadTreeResponse { private readonly subPath: string, private readonly workDir: string, public readonly etag: string, - private readonly filter?: (path: string) => boolean, + private readonly filter?: (path: string, info: { size: number }) => boolean, ) { if (subPath) { if (!subPath.endsWith('/')) { @@ -92,7 +92,7 @@ export class TarArchiveResponse implements ReadTreeResponse { const path = relativePath.slice(this.subPath.length); if (this.filter) { - if (!this.filter(path)) { + if (!this.filter(path, { size: entry.remain })) { entry.resume(); return; } @@ -155,7 +155,7 @@ export class TarArchiveResponse implements ReadTreeResponse { tar.extract({ strip, cwd: dir, - filter: path => { + filter: (path, stat) => { // File path relative to the root extracted directory. Will remove the // top level dir name from the path since its name is hard to predetermine. const relativePath = stripFirstDirectoryFromPath(path); @@ -164,7 +164,7 @@ export class TarArchiveResponse implements ReadTreeResponse { } if (this.filter) { const innerPath = path.split('/').slice(strip).join('/'); - return this.filter(innerPath); + return this.filter(innerPath, { size: stat.size }); } return true; }, diff --git a/packages/backend-common/src/reading/tree/ZipArchiveResponse.test.ts b/packages/backend-common/src/reading/tree/ZipArchiveResponse.test.ts index 659875286e..f6f9a7845a 100644 --- a/packages/backend-common/src/reading/tree/ZipArchiveResponse.test.ts +++ b/packages/backend-common/src/reading/tree/ZipArchiveResponse.test.ts @@ -22,11 +22,15 @@ import { ZipArchiveResponse } from './ZipArchiveResponse'; const archiveData = fs.readFileSync( resolvePath(__filename, '../../__fixtures__/mock-main.zip'), ); +const archiveDataWithExtraDir = fs.readFileSync( + resolvePath(__filename, '../../__fixtures__/mock-with-extra-root-dir.zip'), +); describe('ZipArchiveResponse', () => { beforeEach(() => { mockFs({ '/test-archive.zip': archiveData, + '/test-archive-with-extra-root-dir.zip': archiveDataWithExtraDir, '/tmp': mockFs.directory(), }); }); @@ -58,6 +62,36 @@ describe('ZipArchiveResponse', () => { ]); }); + it('should read files and strip root dir if requested', async () => { + const stream = fs.createReadStream('/test-archive-with-extra-root-dir.zip'); + + const res = new ZipArchiveResponse( + stream, + '', + '/tmp', + 'etag', + undefined, + true, + ); + const files = await res.files(); + + expect(files).toEqual([ + { + path: 'mkdocs.yml', + content: expect.any(Function), + }, + { + path: 'docs/index.md', + content: expect.any(Function), + }, + ]); + const contents = await Promise.all(files.map(f => f.content())); + expect(contents.map(c => c.toString('utf8').trim())).toEqual([ + 'site_name: Test', + '# Test', + ]); + }); + it('should read files with filter', async () => { const stream = fs.createReadStream('/test-archive.zip'); diff --git a/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts b/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts index 45c6880a55..fb22f24b94 100644 --- a/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts +++ b/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts @@ -37,7 +37,8 @@ export class ZipArchiveResponse implements ReadTreeResponse { private readonly subPath: string, private readonly workDir: string, public readonly etag: string, - private readonly filter?: (path: string) => boolean, + private readonly filter?: (path: string, info: { size: number }) => boolean, + private readonly stripFirstDirectory?: boolean, ) { if (subPath) { if (!subPath.endsWith('/')) { @@ -67,7 +68,9 @@ export class ZipArchiveResponse implements ReadTreeResponse { } private shouldBeIncluded(entry: Entry): boolean { - const strippedPath = stripFirstDirectoryFromPath(entry.path); + const strippedPath = this.stripFirstDirectory + ? stripFirstDirectoryFromPath(entry.path) + : entry.path; if (this.subPath) { if (!strippedPath.startsWith(this.subPath)) { @@ -75,7 +78,11 @@ export class ZipArchiveResponse implements ReadTreeResponse { } } if (this.filter) { - return this.filter(this.getInnerPath(entry.path)); + return this.filter(this.getInnerPath(entry.path), { + size: + (entry.vars as { uncompressedSize?: number }).uncompressedSize ?? + entry.vars.compressedSize, + }); } return true; } @@ -95,7 +102,11 @@ export class ZipArchiveResponse implements ReadTreeResponse { if (this.shouldBeIncluded(entry)) { files.push({ - path: this.getInnerPath(stripFirstDirectoryFromPath(entry.path)), + path: this.getInnerPath( + this.stripFirstDirectory + ? stripFirstDirectoryFromPath(entry.path) + : entry.path, + ), content: () => entry.buffer(), }); } else { @@ -144,7 +155,9 @@ export class ZipArchiveResponse implements ReadTreeResponse { // as a zip can have files with directories without directory entries if (entry.type === 'File' && this.shouldBeIncluded(entry)) { const entryPath = this.getInnerPath( - stripFirstDirectoryFromPath(entry.path), + this.stripFirstDirectory + ? stripFirstDirectoryFromPath(entry.path) + : entry.path, ); const dirname = platformPath.dirname(entryPath); if (dirname) { diff --git a/packages/backend-common/src/reading/types.ts b/packages/backend-common/src/reading/types.ts index 8efc833ead..93f287d4fe 100644 --- a/packages/backend-common/src/reading/types.ts +++ b/packages/backend-common/src/reading/types.ts @@ -104,7 +104,7 @@ export type ReadTreeOptions = { * * If no filter is provided all files are extracted. */ - filter?(path: string): boolean; + filter?(path: string, info?: { size: number }): boolean; /** * An etag can be provided to check whether readTree's response has changed from a previous execution. @@ -164,7 +164,7 @@ export type FromArchiveOptions = { // etag of the blob etag: string; // Filter passed on from the ReadTreeOptions - filter?: (path: string) => boolean; + filter?: (path: string, info?: { size: number }) => boolean; }; export interface ReadTreeResponseFactory { diff --git a/packages/backend-test-utils/CHANGELOG.md b/packages/backend-test-utils/CHANGELOG.md index 5671b41fb3..6d33537e7c 100644 --- a/packages/backend-test-utils/CHANGELOG.md +++ b/packages/backend-test-utils/CHANGELOG.md @@ -1,5 +1,13 @@ # @backstage/backend-test-utils +## 0.1.6 + +### Patch Changes + +- Updated dependencies + - @backstage/backend-common@0.9.0 + - @backstage/config@0.1.8 + ## 0.1.5 ### Patch Changes diff --git a/packages/backend-test-utils/package.json b/packages/backend-test-utils/package.json index f1cfffeb37..cd2855b833 100644 --- a/packages/backend-test-utils/package.json +++ b/packages/backend-test-utils/package.json @@ -1,7 +1,7 @@ { "name": "@backstage/backend-test-utils", "description": "Test helpers library for Backstage backends", - "version": "0.1.5", + "version": "0.1.6", "main": "src/index.ts", "types": "src/index.ts", "private": false, @@ -30,9 +30,9 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@backstage/backend-common": "^0.8.9", + "@backstage/backend-common": "^0.9.0", "@backstage/cli": "^0.7.8", - "@backstage/config": "^0.1.5", + "@backstage/config": "^0.1.8", "knex": "^0.95.1", "mysql2": "^2.2.5", "pg": "^8.3.0", diff --git a/packages/backend/CHANGELOG.md b/packages/backend/CHANGELOG.md index 39eabe46b0..b27dbd07e7 100644 --- a/packages/backend/CHANGELOG.md +++ b/packages/backend/CHANGELOG.md @@ -1,5 +1,32 @@ # example-backend +## 0.2.43 + +### Patch Changes + +- Updated dependencies + - @backstage/backend-common@0.9.0 + - @backstage/plugin-catalog-backend@0.13.5 + - @backstage/plugin-search-backend-module-pg@0.1.3 + - @backstage/plugin-auth-backend@0.3.23 + - @backstage/plugin-scaffolder-backend@0.15.2 + - @backstage/integration@0.6.2 + - @backstage/config@0.1.8 + - @backstage/plugin-kubernetes-backend@0.3.15 + - @backstage/plugin-techdocs-backend@0.10.0 + - @backstage/plugin-jenkins-backend@0.1.4 + - @backstage/plugin-app-backend@0.3.16 + - @backstage/plugin-badges-backend@0.1.9 + - @backstage/plugin-code-coverage-backend@0.1.10 + - @backstage/plugin-graphql-backend@0.1.9 + - @backstage/plugin-kafka-backend@0.2.9 + - @backstage/plugin-proxy-backend@0.2.12 + - @backstage/plugin-rollbar-backend@0.1.14 + - @backstage/plugin-scaffolder-backend-module-rails@0.1.5 + - @backstage/plugin-search-backend@0.2.5 + - @backstage/plugin-todo-backend@0.1.11 + - example-app@0.2.43 + ## 0.2.41 ### Patch Changes diff --git a/packages/backend/package.json b/packages/backend/package.json index b2877dedf5..763d891f77 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -1,6 +1,6 @@ { "name": "example-backend", - "version": "0.2.41", + "version": "0.2.43", "main": "dist/index.cjs.js", "types": "src/index.ts", "license": "Apache-2.0", @@ -27,40 +27,42 @@ "migrate:create": "knex migrate:make -x ts" }, "dependencies": { - "@backstage/backend-common": "^0.8.9", + "@backstage/backend-common": "^0.9.0", "@backstage/catalog-client": "^0.3.17", "@backstage/catalog-model": "^0.9.0", - "@backstage/config": "^0.1.5", - "@backstage/integration": "^0.6.0", - "@backstage/plugin-app-backend": "^0.3.15", - "@backstage/plugin-auth-backend": "^0.3.20", - "@backstage/plugin-badges-backend": "^0.1.8", - "@backstage/plugin-catalog-backend": "^0.13.2", - "@backstage/plugin-code-coverage-backend": "^0.1.9", - "@backstage/plugin-graphql-backend": "^0.1.8", - "@backstage/plugin-jenkins-backend": "^0.1.2", - "@backstage/plugin-kubernetes-backend": "^0.3.14", - "@backstage/plugin-kafka-backend": "^0.2.8", - "@backstage/plugin-proxy-backend": "^0.2.9", - "@backstage/plugin-rollbar-backend": "^0.1.13", - "@backstage/plugin-scaffolder-backend": "^0.15.0", - "@backstage/plugin-scaffolder-backend-module-rails": "^0.1.4", - "@backstage/plugin-search-backend": "^0.2.3", + "@backstage/config": "^0.1.8", + "@backstage/integration": "^0.6.2", + "@backstage/plugin-app-backend": "^0.3.16", + "@backstage/plugin-auth-backend": "^0.3.23", + "@backstage/plugin-badges-backend": "^0.1.9", + "@backstage/plugin-catalog-backend": "^0.13.5", + "@backstage/plugin-code-coverage-backend": "^0.1.10", + "@backstage/plugin-graphql-backend": "^0.1.9", + "@backstage/plugin-jenkins-backend": "^0.1.4", + "@backstage/plugin-kubernetes-backend": "^0.3.15", + "@backstage/plugin-kafka-backend": "^0.2.9", + "@backstage/plugin-proxy-backend": "^0.2.12", + "@backstage/plugin-rollbar-backend": "^0.1.14", + "@backstage/plugin-scaffolder-backend": "^0.15.2", + "@backstage/plugin-scaffolder-backend-module-rails": "^0.1.5", + "@backstage/plugin-search-backend": "^0.2.5", "@backstage/plugin-search-backend-node": "^0.4.0", "@backstage/plugin-search-backend-module-elasticsearch": "^0.0.2", - "@backstage/plugin-search-backend-module-pg": "^0.1.1", - "@backstage/plugin-techdocs-backend": "^0.9.2", - "@backstage/plugin-todo-backend": "^0.1.9", + "@backstage/plugin-search-backend-module-pg": "^0.1.3", + "@backstage/plugin-techdocs-backend": "^0.10.0", + "@backstage/plugin-todo-backend": "^0.1.11", "@gitbeaker/node": "^30.2.0", "@octokit/rest": "^18.5.3", "azure-devops-node-api": "^10.2.2", "dockerode": "^3.2.1", - "example-app": "^0.2.41", + "example-app": "^0.2.43", "express": "^4.17.1", "express-promise-router": "^4.1.0", + "express-prom-bundle": "^6.3.6", "knex": "^0.95.1", "pg": "^8.3.0", "pg-connection-string": "^2.3.0", + "prom-client": "^13.2.0", "sqlite3": "^5.0.1", "winston": "^3.2.1" }, diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index aecb233581..78c6c4c20b 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -36,6 +36,7 @@ import { } from '@backstage/backend-common'; import { Config } from '@backstage/config'; import healthcheck from './plugins/healthcheck'; +import { metricsInit, metricsHandler } from './metrics'; import auth from './plugins/auth'; import catalog from './plugins/catalog'; import codeCoverage from './plugins/codecoverage'; @@ -72,6 +73,7 @@ function makeCreateEnv(config: Config) { } async function main() { + metricsInit(); const logger = getRootLogger(); logger.info( @@ -124,6 +126,7 @@ async function main() { const service = createServiceBuilder(module) .loadConfig(config) .addRouter('', await healthcheck(healthcheckEnv)) + .addRouter('', metricsHandler()) .addRouter('/api', apiRouter) .addRouter('', await app(appEnv)); diff --git a/packages/backend/src/metrics.ts b/packages/backend/src/metrics.ts new file mode 100644 index 0000000000..8834022cd9 --- /dev/null +++ b/packages/backend/src/metrics.ts @@ -0,0 +1,63 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { useHotCleanup } from '@backstage/backend-common'; +import { RequestHandler, Request } from 'express'; +import promBundle from 'express-prom-bundle'; +import prom from 'prom-client'; +import * as url from 'url'; + +/** + * Experimental Prometheus metrics used to benchmark the performance of the + * software catalog. Use this at your own risk. + */ +const rootRegEx = new RegExp('^/([^/]*)/.*'); +const apiRegEx = new RegExp('^/api/([^/]*)/.*'); + +function normalizePath(req: Request): string { + const path = url.parse(req.originalUrl || req.url).pathname || '/'; + + // Capture /api/ and the plugin name + if (apiRegEx.test(path)) { + return path.replace(apiRegEx, '/api/$1'); + } + + // Only the first path segment at root level + return path.replace(rootRegEx, '/$1'); +} + +export function metricsInit(): void { + prom.collectDefaultMetrics({ prefix: 'backstage_' }); +} + +/** + * Adds a /metrics endpoint, register default runtime metrics and instrument the router. + */ +export function metricsHandler(): RequestHandler { + // We can only initialize the metrics once and have to clean them up between hot reloads + useHotCleanup(module, () => prom.register.clear()); + + return promBundle({ + includeMethod: true, + includePath: true, + // Using includePath alone is problematic, as it will include path labels with high + // cardinality (e.g. path params). Instead we would have to template them. However, this + // is difficult, as every backend plugin might use different routes. Instead we only take + // the first directory of the path, to have at least an idea how each plugin performs: + normalizePath, + promClient: { collectDefaultMetrics: {} }, + }); +} diff --git a/packages/catalog-model/api-report.md b/packages/catalog-model/api-report.md index 13041d9e32..b9f46add6a 100644 --- a/packages/catalog-model/api-report.md +++ b/packages/catalog-model/api-report.md @@ -230,6 +230,7 @@ export type EntityMeta = JsonObject & { generation?: number; name: string; namespace?: string; + title?: string; description?: string; labels?: Record; annotations?: Record; @@ -722,10 +723,6 @@ export interface TemplateEntityV1beta2 extends Entity { // (undocumented) kind: 'Template'; // (undocumented) - metadata: EntityMeta & { - title?: string; - }; - // (undocumented) spec: { type: string; parameters?: JsonObject | JsonObject[]; diff --git a/packages/catalog-model/src/entity/Entity.ts b/packages/catalog-model/src/entity/Entity.ts index 8843f10f5f..681ac43b80 100644 --- a/packages/catalog-model/src/entity/Entity.ts +++ b/packages/catalog-model/src/entity/Entity.ts @@ -103,7 +103,13 @@ export type EntityMeta = JsonObject & { * The name of the entity. * * Must be unique within the catalog at any given point in time, for any - * given namespace + kind pair. + * given namespace + kind pair. This value is part of the technical + * identifier of the entity, and as such it will appear in URLs, database + * tables, entity references, and similar. It is subject to restrictions + * regarding what characters are allowed. + * + * If you want to use a different, more human readable string with fewer + * restrictions on it in user interfaces, see the `title` field below. */ name: string; @@ -112,6 +118,23 @@ export type EntityMeta = JsonObject & { */ namespace?: string; + /** + * A display name of the entity, to be presented in user interfaces instead + * of the `name` property above, when available. + * + * This field is sometimes useful when the `name` is cumbersome or ends up + * being perceived as overly technical. The title generally does not have + * as stringent format requirements on it, so it may contain special + * characters and be more explanatory. Do keep it very short though, and + * avoid situations where a title can be confused with the name of another + * entity, or where two entities share a title. + * + * Note that this is only for display purposes, and may be ignored by some + * parts of the code. Entity references still always make use of the `name` + * property, not the title. + */ + title?: string; + /** * A short (typically relatively few words, on one line) description of the * entity. diff --git a/packages/catalog-model/src/kinds/TemplateEntityV1beta2.ts b/packages/catalog-model/src/kinds/TemplateEntityV1beta2.ts index 577e5ec9e2..1d7aa2162c 100644 --- a/packages/catalog-model/src/kinds/TemplateEntityV1beta2.ts +++ b/packages/catalog-model/src/kinds/TemplateEntityV1beta2.ts @@ -15,16 +15,13 @@ */ import { JsonObject } from '@backstage/config'; -import type { Entity, EntityMeta } from '../entity/Entity'; +import type { Entity } from '../entity/Entity'; import schema from '../schema/kinds/Template.v1beta2.schema.json'; import { ajvCompiledJsonSchemaValidator } from './util'; export interface TemplateEntityV1beta2 extends Entity { apiVersion: 'backstage.io/v1beta2'; kind: 'Template'; - metadata: EntityMeta & { - title?: string; - }; spec: { type: string; parameters?: JsonObject | JsonObject[]; diff --git a/packages/catalog-model/src/schema/EntityMeta.schema.json b/packages/catalog-model/src/schema/EntityMeta.schema.json index ff0f9c84a8..c834bfb113 100644 --- a/packages/catalog-model/src/schema/EntityMeta.schema.json +++ b/packages/catalog-model/src/schema/EntityMeta.schema.json @@ -53,6 +53,12 @@ "examples": ["default", "admin"], "minLength": 1 }, + "title": { + "type": "string", + "description": "A display name of the entity, to be presented in user interfaces instead of the name property, when available.", + "examples": ["React SSR Template"], + "minLength": 1 + }, "description": { "type": "string", "description": "A short (typically relatively few words, on one line) description of the entity." diff --git a/packages/catalog-model/src/schema/kinds/Template.v1beta2.schema.json b/packages/catalog-model/src/schema/kinds/Template.v1beta2.schema.json index b14cdb8ee0..d68eac936d 100644 --- a/packages/catalog-model/src/schema/kinds/Template.v1beta2.schema.json +++ b/packages/catalog-model/src/schema/kinds/Template.v1beta2.schema.json @@ -69,17 +69,6 @@ "kind": { "enum": ["Template"] }, - "metadata": { - "type": "object", - "properties": { - "title": { - "type": "string", - "description": "The nice display name for the template.", - "examples": ["React SSR Template"], - "minLength": 1 - } - } - }, "spec": { "type": "object", "required": ["type", "steps"], diff --git a/packages/catalog-model/src/validation/entitySchemaValidator.test.ts b/packages/catalog-model/src/validation/entitySchemaValidator.test.ts index 3dfe307f92..b486de1df2 100644 --- a/packages/catalog-model/src/validation/entitySchemaValidator.test.ts +++ b/packages/catalog-model/src/validation/entitySchemaValidator.test.ts @@ -30,6 +30,8 @@ describe('entitySchemaValidator', () => { generation: 13, name: 'test', namespace: 'ns', + title: 'My Component, Yay', + description: 'Yeah this is probably the best component so far', labels: { 'backstage.io/custom': 'ValueStuff', }, @@ -192,6 +194,21 @@ describe('entitySchemaValidator', () => { expect(() => validator(entity)).toThrow(/namespace/); }); + it('accepts missing title', () => { + delete entity.metadata.title; + expect(() => validator(entity)).not.toThrow(); + }); + + it('rejects bad title type', () => { + entity.metadata.title = 7; + expect(() => validator(entity)).toThrow(/title/); + }); + + it('rejects empty title', () => { + entity.metadata.title = ''; + expect(() => validator(entity)).toThrow(/title/); + }); + it('accepts missing description', () => { delete entity.metadata.description; expect(() => validator(entity)).not.toThrow(); diff --git a/packages/cli/CHANGELOG.md b/packages/cli/CHANGELOG.md index 848a106db0..064438d17e 100644 --- a/packages/cli/CHANGELOG.md +++ b/packages/cli/CHANGELOG.md @@ -1,5 +1,20 @@ # @backstage/cli +## 0.7.9 + +### Patch Changes + +- f3bba3d2b: Remove debug logging +- 8ea1e96b3: Fix file path handling in diff commands on Windows. +- 2518aab58: Compensate for error formatting mismatch between Webpack 5 and react-dev-utils +- 1ac2961c3: Reintroduce Node.js shims that were removed in the Webpack 5 migration. +- 8d07a8b03: Add Buffer to `ProvidePlugin` since this is no longer provided in `webpack@5` +- fe506a0cf: Remove Webpack deprecation message when running build. +- 485438a56: Fix `backstage-cli backend:dev` argument passing +- Updated dependencies + - @backstage/config@0.1.7 + - @backstage/config-loader@0.6.7 + ## 0.7.8 ### Patch Changes diff --git a/packages/cli/package.json b/packages/cli/package.json index 63b8db373e..13d146de72 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,7 +1,7 @@ { "name": "@backstage/cli", "description": "CLI for developing Backstage plugins and apps", - "version": "0.7.8", + "version": "0.7.9", "private": false, "publishConfig": { "access": "public" @@ -31,8 +31,8 @@ "@babel/core": "^7.4.4", "@babel/plugin-transform-modules-commonjs": "^7.4.4", "@backstage/cli-common": "^0.1.2", - "@backstage/config": "^0.1.6", - "@backstage/config-loader": "^0.6.6", + "@backstage/config": "^0.1.8", + "@backstage/config-loader": "^0.6.7", "@hot-loader/react-dom": "^16.13.0", "@lerna/package-graph": "^4.0.0", "@lerna/project": "^4.0.0", @@ -56,6 +56,7 @@ "@yarnpkg/lockfile": "^1.1.0", "babel-plugin-dynamic-import-node": "^2.3.3", "bfj": "^7.0.2", + "buffer": "^6.0.3", "chalk": "^4.0.0", "chokidar": "^3.3.1", "commander": "^6.1.0", @@ -84,6 +85,7 @@ "json-schema": "^0.3.0", "lodash": "^4.17.19", "mini-css-extract-plugin": "^1.4.1", + "node-libs-browser": "^2.2.1", "ora": "^5.3.0", "postcss": "^8.1.0", "process": "^0.11.10", @@ -118,14 +120,14 @@ "yn": "^4.0.0" }, "devDependencies": { - "@backstage/backend-common": "^0.8.9", - "@backstage/config": "^0.1.6", - "@backstage/core-components": "^0.3.1", + "@backstage/backend-common": "^0.9.0", + "@backstage/config": "^0.1.8", + "@backstage/core-components": "^0.3.3", "@backstage/core-plugin-api": "^0.1.6", - "@backstage/core-app-api": "^0.1.8", - "@backstage/dev-utils": "^0.2.6", + "@backstage/core-app-api": "^0.1.10", + "@backstage/dev-utils": "^0.2.7", "@backstage/test-utils": "^0.1.17", - "@backstage/theme": "^0.2.9", + "@backstage/theme": "^0.2.10", "@types/diff": "^5.0.0", "@types/express": "^4.17.6", "@types/fs-extra": "^9.0.1", diff --git a/packages/cli/src/lib/bundler/backend.ts b/packages/cli/src/lib/bundler/backend.ts index 567fc3476a..bd4ab42098 100644 --- a/packages/cli/src/lib/bundler/backend.ts +++ b/packages/cli/src/lib/bundler/backend.ts @@ -28,7 +28,6 @@ export async function serveBackend(options: BackendServeOptions) { const compiler = webpack(config, (err: Error | undefined) => { if (err) { - console.log('here'); console.error(err); } else console.log('Build succeeded'); }); diff --git a/packages/cli/src/lib/bundler/bundle.ts b/packages/cli/src/lib/bundler/bundle.ts index 082b1247ab..2f23d5cae6 100644 --- a/packages/cli/src/lib/bundler/bundle.ts +++ b/packages/cli/src/lib/bundler/bundle.ts @@ -117,9 +117,22 @@ async function build(compiler: webpack.Compiler, isCi: boolean) { if (!stats) { throw new Error('No stats provided'); } - const { errors, warnings } = formatWebpackMessages( - stats.toJson({ all: false, warnings: true, errors: true }), - ); + + const serializedStats = stats.toJson({ + all: false, + warnings: true, + errors: true, + }); + // NOTE(freben): The code below that extracts the message part of the errors, + // is due to react-dev-utils not yet being compatible with webpack 5. This + // may be possible to remove (just passing the serialized stats object + // directly into the format function) after a new release of react-dev-utils + // has been made available. + // See https://github.com/facebook/create-react-app/issues/9880 + const { errors, warnings } = formatWebpackMessages({ + errors: serializedStats.errors?.map(e => (e.message ? e.message : e)), + warnings: serializedStats.warnings?.map(e => (e.message ? e.message : e)), + }); if (errors.length) { // Only keep the first error. Others are often indicative diff --git a/packages/cli/src/lib/bundler/config.ts b/packages/cli/src/lib/bundler/config.ts index f749c70ed2..6e3f445f4a 100644 --- a/packages/cli/src/lib/bundler/config.ts +++ b/packages/cli/src/lib/bundler/config.ts @@ -32,6 +32,7 @@ import { BundlingOptions, BackendBundlingOptions, LernaPackage } from './types'; import { version } from '../../lib/version'; import { paths as cliPaths } from '../../lib/paths'; import { runPlain } from '../run'; +import pickBy from 'lodash/pickBy'; export function resolveBaseUrl(config: Config): URL { const baseUrl = config.getString('app.baseUrl'); @@ -119,6 +120,7 @@ export async function createConfig( plugins.push( new ProvidePlugin({ process: 'process/browser', + Buffer: ['buffer', 'Buffer'], }), ); @@ -176,6 +178,7 @@ export async function createConfig( extensions: ['.ts', '.tsx', '.mjs', '.js', '.jsx'], mainFields: ['browser', 'module', 'main'], fallback: { + ...pickBy(require('node-libs-browser')), module: false, dgram: false, dns: false, @@ -208,7 +211,7 @@ export async function createConfig( output: { path: paths.targetDist, publicPath: validBaseUrl.pathname, - filename: isDev ? '[name].js' : 'static/[name].[hash:8].js', + filename: isDev ? '[name].js' : 'static/[name].[fullhash:8].js', chunkFilename: isDev ? '[name].chunk.js' : 'static/[name].[chunkhash:8].chunk.js', @@ -317,6 +320,7 @@ export async function createBackendConfig( new RunScriptWebpackPlugin({ name: 'main.js', nodeArgs: options.inspectEnabled ? ['--inspect'] : undefined, + args: process.argv.slice(3), // drop `node backstage-cli backend:dev` }), new webpack.HotModuleReplacementPlugin(), ...(checksEnabled diff --git a/packages/cli/src/lib/diff/handlers.ts b/packages/cli/src/lib/diff/handlers.ts index 5cbddde0ec..4a17125e5a 100644 --- a/packages/cli/src/lib/diff/handlers.ts +++ b/packages/cli/src/lib/diff/handlers.ts @@ -16,6 +16,7 @@ import chalk from 'chalk'; import { diffLines } from 'diff'; +import { sep, posix } from 'path'; import { FileDiff, PromptFunc, FileHandler, WriteFileFunc } from './types'; function sortObjectKeys(obj: Record) { @@ -281,7 +282,7 @@ export async function handleAllFiles( promptFunc: PromptFunc, ) { for (const file of files) { - const { path } = file; + const path = file.path.split(sep).join(posix.sep); const fileHandler = fileHandlers.find(handler => handler.patterns.some(pattern => typeof pattern === 'string' ? pattern === path : pattern.test(path), diff --git a/packages/codemods/CHANGELOG.md b/packages/codemods/CHANGELOG.md index 192e6810bc..74a3022bf5 100644 --- a/packages/codemods/CHANGELOG.md +++ b/packages/codemods/CHANGELOG.md @@ -1,5 +1,21 @@ # @backstage/codemods +## 0.1.11 + +### Patch Changes + +- Updated dependencies + - @backstage/core-app-api@0.1.10 + - @backstage/core-components@0.3.3 + +## 0.1.10 + +### Patch Changes + +- Updated dependencies + - @backstage/core-app-api@0.1.9 + - @backstage/core-components@0.3.2 + ## 0.1.9 ### Patch Changes diff --git a/packages/codemods/package.json b/packages/codemods/package.json index 8f87631897..04bf0ac014 100644 --- a/packages/codemods/package.json +++ b/packages/codemods/package.json @@ -1,7 +1,7 @@ { "name": "@backstage/codemods", "description": "A collection of codemods for Backstage projects", - "version": "0.1.9", + "version": "0.1.11", "private": false, "publishConfig": { "access": "public", diff --git a/packages/config-loader/CHANGELOG.md b/packages/config-loader/CHANGELOG.md index 44819b6903..aee4cf453f 100644 --- a/packages/config-loader/CHANGELOG.md +++ b/packages/config-loader/CHANGELOG.md @@ -1,5 +1,14 @@ # @backstage/config-loader +## 0.6.7 + +### Patch Changes + +- 0ade9d02b: Include `devDependencies` and `optionalDependencies` in the detection of Backstage packages when collecting configuration schema. +- 9b8cec063: Add support for config file watching through a new group of `watch` options to `loadConfig`. +- Updated dependencies + - @backstage/config@0.1.7 + ## 0.6.6 ### Patch Changes diff --git a/packages/config-loader/api-report.md b/packages/config-loader/api-report.md index 47bc22c60f..38aa0b1ebb 100644 --- a/packages/config-loader/api-report.md +++ b/packages/config-loader/api-report.md @@ -37,6 +37,10 @@ export type LoadConfigOptions = { configPaths: string[]; env?: string; experimentalEnvFunc?: EnvFunc; + watch?: { + onChange: (configs: AppConfig[]) => void; + stopSignal?: Promise; + }; }; // Warning: (ae-forgotten-export) The symbol "Options" needs to be exported by the entry point index.d.ts diff --git a/packages/config-loader/package.json b/packages/config-loader/package.json index 33b8acc3b9..f810841d3e 100644 --- a/packages/config-loader/package.json +++ b/packages/config-loader/package.json @@ -1,7 +1,7 @@ { "name": "@backstage/config-loader", "description": "Config loading functionality used by Backstage backend, and CLI", - "version": "0.6.6", + "version": "0.6.7", "private": false, "publishConfig": { "access": "public", @@ -31,9 +31,10 @@ }, "dependencies": { "@backstage/cli-common": "^0.1.1", - "@backstage/config": "^0.1.6", + "@backstage/config": "^0.1.7", "@types/json-schema": "^7.0.6", "ajv": "^7.0.3", + "chokidar": "^3.5.2", "fs-extra": "9.1.0", "json-schema": "^0.3.0", "json-schema-merge-allof": "^0.8.1", diff --git a/packages/config-loader/src/lib/schema/collect.test.ts b/packages/config-loader/src/lib/schema/collect.test.ts index 479e63c94c..ef9d38faa9 100644 --- a/packages/config-loader/src/lib/schema/collect.test.ts +++ b/packages/config-loader/src/lib/schema/collect.test.ts @@ -87,6 +87,8 @@ describe('collectConfigSchemas', () => { dependencies: { c1: '0.0.0', c2: '0.0.0', + }, + devDependencies: { '@backstage/mock': '0.0.0', }, configSchema: { ...mockSchema, title: 'b' }, diff --git a/packages/config-loader/src/lib/schema/collect.ts b/packages/config-loader/src/lib/schema/collect.ts index b53389d499..b363e8e5f3 100644 --- a/packages/config-loader/src/lib/schema/collect.ts +++ b/packages/config-loader/src/lib/schema/collect.ts @@ -74,6 +74,8 @@ export async function collectConfigSchemas( const pkg = await fs.readJson(pkgPath); const depNames = [ ...Object.keys(pkg.dependencies ?? {}), + ...Object.keys(pkg.devDependencies ?? {}), + ...Object.keys(pkg.optionalDependencies ?? {}), ...Object.keys(pkg.peerDependencies ?? {}), ]; diff --git a/packages/config-loader/src/loader.test.ts b/packages/config-loader/src/loader.test.ts index b0e236f9c9..3f51e76936 100644 --- a/packages/config-loader/src/loader.test.ts +++ b/packages/config-loader/src/loader.test.ts @@ -14,11 +14,13 @@ * limitations under the License. */ +import { AppConfig } from '@backstage/config'; import { loadConfig } from './loader'; import mockFs from 'mock-fs'; +import fs from 'fs-extra'; describe('loadConfig', () => { - beforeAll(() => { + beforeEach(() => { process.env.MY_SECRET = 'is-secret'; process.env.SUBSTITUTE_ME = 'substituted'; @@ -63,7 +65,7 @@ describe('loadConfig', () => { }); }); - afterAll(() => { + afterEach(() => { mockFs.restore(); }); @@ -170,4 +172,81 @@ describe('loadConfig', () => { }, ]); }); + + it('watches config files', async () => { + const onChange = defer(); + const stopSignal = defer(); + + await expect( + loadConfig({ + configRoot: '/root', + configPaths: [], + watch: { + onChange: onChange.resolve, + stopSignal: stopSignal.promise, + }, + }), + ).resolves.toEqual([ + { + context: 'app-config.yaml', + data: { + app: { + title: 'Example App', + sessionKey: 'abc123', + escaped: '${Escaped}', + }, + }, + }, + ]); + + await fs.writeJson('/root/app-config.yaml', { + app: { + title: 'New Title', + }, + }); + await expect(onChange.promise).resolves.toEqual([ + { + context: 'app-config.yaml', + data: { + app: { + title: 'New Title', + }, + }, + }, + ]); + + stopSignal.resolve(); + }); + + it('stops watching config files', async () => { + const stopSignal = defer(); + + await loadConfig({ + configRoot: '/root', + configPaths: [], + watch: { + onChange: () => { + expect('not').toBe('called'); + }, + stopSignal: stopSignal.promise, + }, + }); + + stopSignal.resolve(); + + await fs.writeJson('/root/app-config.yaml', { + app: { + title: 'New Title', + }, + }); + await new Promise(resolve => setTimeout(resolve, 1000)); + }); + + function defer() { + let resolve: (value: T) => void; + const promise = new Promise(_resolve => { + resolve = _resolve; + }); + return { promise, resolve: resolve! }; + } }); diff --git a/packages/config-loader/src/loader.ts b/packages/config-loader/src/loader.ts index 3846d6474b..4dbfc007a7 100644 --- a/packages/config-loader/src/loader.ts +++ b/packages/config-loader/src/loader.ts @@ -16,6 +16,7 @@ import fs from 'fs-extra'; import yaml from 'yaml'; +import chokidar from 'chokidar'; import { resolve as resolvePath, dirname, isAbsolute, basename } from 'path'; import { AppConfig } from '@backstage/config'; import { @@ -42,13 +43,27 @@ export type LoadConfigOptions = { * @experimental This API is not stable and may change at any point */ experimentalEnvFunc?: EnvFunc; + + /** + * An optional configuration that enables watching of config files. + */ + watch?: { + /** + * A listener that is called when a config file is changed. + */ + onChange: (configs: AppConfig[]) => void; + + /** + * An optional signal that stops the watcher once the promise resolves. + */ + stopSignal?: Promise; + }; }; export async function loadConfig( options: LoadConfigOptions, ): Promise { - const configs = []; - const { configRoot, experimentalEnvFunc: envFunc } = options; + const { configRoot, experimentalEnvFunc: envFunc, watch } = options; const configPaths = options.configPaths.slice(); // If no paths are provided, we default to reading @@ -64,7 +79,9 @@ export async function loadConfig( const env = envFunc ?? (async (name: string) => process.env[name]); - try { + const loadConfigFiles = async () => { + const configs = []; + for (const configPath of configPaths) { if (!isAbsolute(configPath)) { throw new Error(`Config load path is not absolute: '${configPath}'`); @@ -83,13 +100,50 @@ export async function loadConfig( configs.push({ data, context: basename(configPath) }); } + + return configs; + }; + + let fileConfigs; + try { + fileConfigs = await loadConfigFiles(); } catch (error) { throw new Error( `Failed to read static configuration file, ${error.message}`, ); } - configs.push(...readEnvConfig(process.env)); + const envConfigs = await readEnvConfig(process.env); - return configs; + // Set up config file watching if requested by the caller + if (watch) { + let currentSerializedConfig = JSON.stringify(fileConfigs); + + const watcher = chokidar.watch(configPaths, { + usePolling: process.env.NODE_ENV === 'test', + }); + watcher.on('change', async () => { + try { + const newConfigs = await loadConfigFiles(); + const newSerializedConfig = JSON.stringify(newConfigs); + + if (currentSerializedConfig === newSerializedConfig) { + return; + } + currentSerializedConfig = newSerializedConfig; + + watch.onChange([...newConfigs, ...envConfigs]); + } catch (error) { + console.error(`Failed to reload configuration files, ${error}`); + } + }); + + if (watch.stopSignal) { + watch.stopSignal.then(() => { + watcher.close(); + }); + } + } + + return [...fileConfigs, ...envConfigs]; } diff --git a/packages/config/CHANGELOG.md b/packages/config/CHANGELOG.md index 63aa34b034..742ae86a57 100644 --- a/packages/config/CHANGELOG.md +++ b/packages/config/CHANGELOG.md @@ -1,5 +1,17 @@ # @backstage/config +## 0.1.8 + +### Patch Changes + +- 47113f1f1: Only warn once per key when trying to read visibility-filtered values + +## 0.1.7 + +### Patch Changes + +- 90f25476a: Extended the `Config` interface to have an optional `subscribe` method that can be used be notified of updates to the configuration. + ## 0.1.6 ### Patch Changes diff --git a/packages/config/api-report.md b/packages/config/api-report.md index eff4236026..91f37894c0 100644 --- a/packages/config/api-report.md +++ b/packages/config/api-report.md @@ -16,6 +16,9 @@ export type AppConfig = { // // @public (undocumented) export type Config = { + subscribe?(onChange: () => void): { + unsubscribe: () => void; + }; has(key: string): boolean; keys(): string[]; get(key?: string): T; diff --git a/packages/config/package.json b/packages/config/package.json index 5d9bf94446..6adee62ca3 100644 --- a/packages/config/package.json +++ b/packages/config/package.json @@ -1,7 +1,7 @@ { "name": "@backstage/config", "description": "Config API used by Backstage core, backend, and CLI", - "version": "0.1.6", + "version": "0.1.8", "private": false, "publishConfig": { "access": "public", diff --git a/packages/config/src/reader.test.ts b/packages/config/src/reader.test.ts index 2f846fe159..7f3513abf6 100644 --- a/packages/config/src/reader.test.ts +++ b/packages/config/src/reader.test.ts @@ -195,7 +195,7 @@ describe('ConfigReader', () => { { data: DATA, context: CTX, - filteredKeys: ['a', 'b[0]'], + filteredKeys: ['a', 'a2', 'b[0]'], }, ]); @@ -204,13 +204,13 @@ describe('ConfigReader', () => { "Failed to read configuration value at 'a' as it is not visible. See https://backstage.io/docs/conf/defining#visibility for instructions on how to make it visible.", ], }); - expect(withLogCollector(() => config.getOptionalString('a'))).toMatchObject( - { - warn: [ - "Failed to read configuration value at 'a' as it is not visible. See https://backstage.io/docs/conf/defining#visibility for instructions on how to make it visible.", - ], - }, - ); + expect( + withLogCollector(() => config.getOptionalString('a2')), + ).toMatchObject({ + warn: [ + "Failed to read configuration value at 'a2' as it is not visible. See https://backstage.io/docs/conf/defining#visibility for instructions on how to make it visible.", + ], + }); expect( withLogCollector(() => config.getOptionalConfigArray('b')), ).toMatchObject({ @@ -222,6 +222,30 @@ describe('ConfigReader', () => { (process.env as any).NODE_ENV = oldEnv; }); + it('only warns once when accessing filtered keys in development mode', () => { + const oldEnv = process.env.NODE_ENV; + (process.env as any).NODE_ENV = 'development'; + + const config = ConfigReader.fromConfigs([ + { + data: DATA, + context: CTX, + filteredKeys: ['a'], + }, + ]); + + expect(withLogCollector(() => config.getOptional('a'))).toMatchObject({ + warn: [ + "Failed to read configuration value at 'a' as it is not visible. See https://backstage.io/docs/conf/defining#visibility for instructions on how to make it visible.", + ], + }); + expect(withLogCollector(() => config.getOptional('a'))).toMatchObject({ + warn: [], + }); + + (process.env as any).NODE_ENV = oldEnv; + }); + it('should not warn when accessing filtered keys outside of development mode', () => { const config = ConfigReader.fromConfigs([ { diff --git a/packages/config/src/reader.ts b/packages/config/src/reader.ts index d93706c12e..c6f69e5b29 100644 --- a/packages/config/src/reader.ts +++ b/packages/config/src/reader.ts @@ -63,6 +63,7 @@ export class ConfigReader implements Config { * the frontend in development mode. */ private filteredKeys?: string[]; + private notifiedFilteredKeys = new Set(); static fromConfigs(configs: AppConfig[]): ConfigReader { if (configs.length === 0) { @@ -118,7 +119,11 @@ export class ConfigReader implements Config { if (process.env.NODE_ENV === 'development') { if (fallbackValue === undefined && key) { const fullKey = this.fullKey(key); - if (this.filteredKeys?.includes(fullKey)) { + if ( + this.filteredKeys?.includes(fullKey) && + !this.notifiedFilteredKeys.has(fullKey) + ) { + this.notifiedFilteredKeys.add(fullKey); // eslint-disable-next-line no-console console.warn( `Failed to read configuration value at '${fullKey}' as it is not visible. ` + @@ -190,7 +195,11 @@ export class ConfigReader implements Config { if (!configs) { if (process.env.NODE_ENV === 'development') { const fullKey = this.fullKey(key); - if (this.filteredKeys?.some(k => k.startsWith(fullKey))) { + if ( + this.filteredKeys?.some(k => k.startsWith(fullKey)) && + !this.notifiedFilteredKeys.has(key) + ) { + this.notifiedFilteredKeys.add(key); // eslint-disable-next-line no-console console.warn( `Failed to read configuration array at '${key}' as it does not have any visible elements. ` + @@ -310,7 +319,11 @@ export class ConfigReader implements Config { if (value === undefined) { if (process.env.NODE_ENV === 'development') { const fullKey = this.fullKey(key); - if (this.filteredKeys?.includes(fullKey)) { + if ( + this.filteredKeys?.includes(fullKey) && + !this.notifiedFilteredKeys.has(fullKey) + ) { + this.notifiedFilteredKeys.add(fullKey); // eslint-disable-next-line no-console console.warn( `Failed to read configuration value at '${fullKey}' as it is not visible. ` + diff --git a/packages/config/src/types.ts b/packages/config/src/types.ts index 1135e7d554..84de5dae70 100644 --- a/packages/config/src/types.ts +++ b/packages/config/src/types.ts @@ -26,6 +26,17 @@ export type AppConfig = { }; export type Config = { + /** + * Subscribes to the configuration object in order to receive a notification + * whenever any value within the configuration has changed. + * + * This method is optional to implement, and consumers need to check if it is + * implemented before invoking it. + */ + subscribe?(onChange: () => void): { + unsubscribe: () => void; + }; + has(key: string): boolean; keys(): string[]; diff --git a/packages/core-app-api/CHANGELOG.md b/packages/core-app-api/CHANGELOG.md index 1ec944ac4a..db5158a697 100644 --- a/packages/core-app-api/CHANGELOG.md +++ b/packages/core-app-api/CHANGELOG.md @@ -1,5 +1,30 @@ # @backstage/core-app-api +## 0.1.10 + +### Patch Changes + +- cfcb486aa: Add system icons for the built-in entity types and use them in the entity list of the `catalog-import` plugin. +- 392b36fa1: Added support for using authenticating via GitHub Apps in addition to GitHub OAuth Apps. It used to be possible to use GitHub Apps, but they did not handle session refresh correctly. + + Note that GitHub Apps handle OAuth scope at the app installation level, meaning that the `scope` parameter for `getAccessToken` has no effect. When calling `getAccessToken` in open source plugins, one should still include the appropriate scope, but also document in the plugin README what scopes are required in the case of GitHub Apps. + + In addition, the `authHandler` and `signInResolver` options have been implemented for the GitHub provider in the auth backend. + +- Updated dependencies + - @backstage/core-components@0.3.3 + - @backstage/config@0.1.8 + +## 0.1.9 + +### Patch Changes + +- 72a31c29a: Add support for additional app origins +- Updated dependencies + - @backstage/config@0.1.7 + - @backstage/core-components@0.3.2 + - @backstage/theme@0.2.10 + ## 0.1.8 ### Patch Changes diff --git a/packages/core-app-api/api-report.md b/packages/core-app-api/api-report.md index e2c6b3a939..20d8a5a770 100644 --- a/packages/core-app-api/api-report.md +++ b/packages/core-app-api/api-report.md @@ -394,7 +394,7 @@ export type GithubSession = { providerInfo: { accessToken: string; scopes: Set; - expiresAt: Date; + expiresAt?: Date; }; profile: ProfileInfo; backstageIdentity: BackstageIdentity; diff --git a/packages/core-app-api/package.json b/packages/core-app-api/package.json index 2b66a991f1..90c962be3e 100644 --- a/packages/core-app-api/package.json +++ b/packages/core-app-api/package.json @@ -1,7 +1,7 @@ { "name": "@backstage/core-app-api", "description": "Core app API used by Backstage apps", - "version": "0.1.8", + "version": "0.1.10", "private": false, "publishConfig": { "access": "public", @@ -29,10 +29,10 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@backstage/core-components": "^0.3.1", - "@backstage/config": "^0.1.6", + "@backstage/core-components": "^0.3.3", + "@backstage/config": "^0.1.8", "@backstage/core-plugin-api": "^0.1.6", - "@backstage/theme": "^0.2.9", + "@backstage/theme": "^0.2.10", "@material-ui/core": "^4.12.2", "@material-ui/icons": "^4.9.1", "@types/react": "*", @@ -44,7 +44,7 @@ "zen-observable": "^0.8.15" }, "devDependencies": { - "@backstage/cli": "^0.7.8", + "@backstage/cli": "^0.7.9", "@backstage/test-utils": "^0.1.17", "@backstage/test-utils-core": "^0.1.2", "@testing-library/jest-dom": "^5.10.1", diff --git a/packages/core-app-api/src/apis/implementations/auth/github/GithubAuth.ts b/packages/core-app-api/src/apis/implementations/auth/github/GithubAuth.ts index 21ebc5699a..85e3e79ad5 100644 --- a/packages/core-app-api/src/apis/implementations/auth/github/GithubAuth.ts +++ b/packages/core-app-api/src/apis/implementations/auth/github/GithubAuth.ts @@ -29,15 +29,17 @@ import { import { SessionManager } from '../../../../lib/AuthSessionManager/types'; import { AuthSessionStore, + RefreshingAuthSessionManager, StaticAuthSessionManager, } from '../../../../lib/AuthSessionManager'; import { OAuthApiCreateOptions } from '../types'; +import { OptionalRefreshSessionManagerMux } from '../../../../lib/AuthSessionManager/OptionalRefreshSessionManagerMux'; export type GithubAuthResponse = { providerInfo: { accessToken: string; scope: string; - expiresInSeconds: number; + expiresInSeconds?: number; }; profile: ProfileInfo; backstageIdentity: BackstageIdentity; @@ -68,27 +70,46 @@ class GithubAuth implements OAuthApi, SessionApi { providerInfo: { accessToken: res.providerInfo.accessToken, scopes: GithubAuth.normalizeScope(res.providerInfo.scope), - expiresAt: new Date( - Date.now() + res.providerInfo.expiresInSeconds * 1000, - ), + expiresAt: res.providerInfo.expiresInSeconds + ? new Date(Date.now() + res.providerInfo.expiresInSeconds * 1000) + : undefined, }, }; }, }); - const sessionManager = new StaticAuthSessionManager({ + const refreshingSessionManager = new RefreshingAuthSessionManager({ connector, defaultScopes: new Set(defaultScopes), sessionScopes: (session: GithubSession) => session.providerInfo.scopes, + sessionShouldRefresh: (session: GithubSession) => { + const { expiresAt } = session.providerInfo; + if (!expiresAt) { + return false; + } + const expiresInSec = (expiresAt.getTime() - Date.now()) / 1000; + return expiresInSec < 60 * 5; + }, }); - const authSessionStore = new AuthSessionStore({ - manager: sessionManager, + const staticSessionManager = new AuthSessionStore({ + manager: new StaticAuthSessionManager({ + connector, + defaultScopes: new Set(defaultScopes), + sessionScopes: (session: GithubSession) => session.providerInfo.scopes, + }), storageKey: `${provider.id}Session`, sessionScopes: (session: GithubSession) => session.providerInfo.scopes, }); - return new GithubAuth(authSessionStore); + const sessionManagerMux = new OptionalRefreshSessionManagerMux({ + refreshingSessionManager, + staticSessionManager, + sessionCanRefresh: session => + session.providerInfo.expiresAt !== undefined, + }); + + return new GithubAuth(sessionManagerMux); } constructor(private readonly sessionManager: SessionManager) {} diff --git a/packages/core-app-api/src/apis/implementations/auth/github/types.ts b/packages/core-app-api/src/apis/implementations/auth/github/types.ts index 88df25b49d..f5dae3a064 100644 --- a/packages/core-app-api/src/apis/implementations/auth/github/types.ts +++ b/packages/core-app-api/src/apis/implementations/auth/github/types.ts @@ -20,7 +20,7 @@ export type GithubSession = { providerInfo: { accessToken: string; scopes: Set; - expiresAt: Date; + expiresAt?: Date; }; profile: ProfileInfo; backstageIdentity: BackstageIdentity; diff --git a/packages/core-app-api/src/app/icons.tsx b/packages/core-app-api/src/app/icons.tsx index bf45155096..a102026b29 100644 --- a/packages/core-app-api/src/app/icons.tsx +++ b/packages/core-app-api/src/app/icons.tsx @@ -15,17 +15,22 @@ */ import { IconComponent } from '@backstage/core-plugin-api'; -import MuiMenuBookIcon from '@material-ui/icons/MenuBook'; +import MuiApartmentIcon from '@material-ui/icons/Apartment'; import MuiBrokenImageIcon from '@material-ui/icons/BrokenImage'; +import MuiCategoryIcon from '@material-ui/icons/Category'; import MuiChatIcon from '@material-ui/icons/Chat'; import MuiDashboardIcon from '@material-ui/icons/Dashboard'; +import MuiDocsIcon from '@material-ui/icons/Description'; import MuiEmailIcon from '@material-ui/icons/Email'; +import MuiExtensionIcon from '@material-ui/icons/Extension'; import MuiGitHubIcon from '@material-ui/icons/GitHub'; import MuiHelpIcon from '@material-ui/icons/Help'; +import MuiLocationOnIcon from '@material-ui/icons/LocationOn'; +import MuiMemoryIcon from '@material-ui/icons/Memory'; +import MuiMenuBookIcon from '@material-ui/icons/MenuBook'; import MuiPeopleIcon from '@material-ui/icons/People'; import MuiPersonIcon from '@material-ui/icons/Person'; import MuiWarningIcon from '@material-ui/icons/Warning'; -import MuiDocsIcon from '@material-ui/icons/Description'; type AppIconsKey = | 'brokenImage' @@ -37,6 +42,13 @@ type AppIconsKey = | 'github' | 'group' | 'help' + | 'kind:api' + | 'kind:component' + | 'kind:domain' + | 'kind:group' + | 'kind:location' + | 'kind:system' + | 'kind:user' | 'user' | 'warning'; @@ -53,6 +65,13 @@ export const defaultAppIcons: AppIcons = { github: MuiGitHubIcon, group: MuiPeopleIcon, help: MuiHelpIcon, + 'kind:api': MuiExtensionIcon, + 'kind:component': MuiMemoryIcon, + 'kind:domain': MuiApartmentIcon, + 'kind:group': MuiPeopleIcon, + 'kind:location': MuiLocationOnIcon, + 'kind:system': MuiCategoryIcon, + 'kind:user': MuiPersonIcon, user: MuiPersonIcon, warning: MuiWarningIcon, }; diff --git a/packages/core-app-api/src/lib/AuthConnector/DefaultAuthConnector.test.ts b/packages/core-app-api/src/lib/AuthConnector/DefaultAuthConnector.test.ts index ce5df9fc91..28125f6fa8 100644 --- a/packages/core-app-api/src/lib/AuthConnector/DefaultAuthConnector.test.ts +++ b/packages/core-app-api/src/lib/AuthConnector/DefaultAuthConnector.test.ts @@ -126,7 +126,7 @@ describe('DefaultAuthConnector', () => { expect(popupSpy).toBeCalledTimes(1); expect(popupSpy.mock.calls[0][0]).toMatchObject({ - url: 'http://my-host/api/auth/my-provider/start?scope=a%20b&env=production', + url: 'http://my-host/api/auth/my-provider/start?scope=a%20b&origin=http%3A%2F%2Flocalhost&env=production', }); await expect(sessionPromise).resolves.toEqual({ @@ -174,7 +174,7 @@ describe('DefaultAuthConnector', () => { expect(popupSpy).toBeCalledTimes(1); expect(popupSpy.mock.calls[0][0]).toMatchObject({ - url: 'http://my-host/api/auth/my-provider/start?scope=-ab-&env=production', + url: 'http://my-host/api/auth/my-provider/start?scope=-ab-&origin=http%3A%2F%2Flocalhost&env=production', }); }); }); diff --git a/packages/core-app-api/src/lib/AuthConnector/DefaultAuthConnector.ts b/packages/core-app-api/src/lib/AuthConnector/DefaultAuthConnector.ts index d707acdc9b..261a008d4b 100644 --- a/packages/core-app-api/src/lib/AuthConnector/DefaultAuthConnector.ts +++ b/packages/core-app-api/src/lib/AuthConnector/DefaultAuthConnector.ts @@ -152,7 +152,10 @@ export class DefaultAuthConnector private async showPopup(scopes: Set): Promise { const scope = this.joinScopesFunc(scopes); - const popupUrl = await this.buildUrl('/start', { scope }); + const popupUrl = await this.buildUrl('/start', { + scope, + origin: location.origin, + }); const payload = await showLoginPopup({ url: popupUrl, diff --git a/packages/core-app-api/src/lib/AuthSessionManager/AuthSessionStore.test.ts b/packages/core-app-api/src/lib/AuthSessionManager/AuthSessionStore.test.ts index 4ceadd51ba..ded26c59e5 100644 --- a/packages/core-app-api/src/lib/AuthSessionManager/AuthSessionStore.test.ts +++ b/packages/core-app-api/src/lib/AuthSessionManager/AuthSessionStore.test.ts @@ -128,6 +128,19 @@ describe('GheAuth AuthSessionStore', () => { expect(manager.removeSession).toHaveBeenCalled(); }); + it('should set session', async () => { + const manager = new MockManager(); + const store = new AuthSessionStore({ manager, ...defaultOptions }); + + await expect(store.getSession({ optional: true })).resolves.toBe(undefined); + expect(localStorage.getItem('my-key')).toBe(null); + expect(manager.setSession).not.toHaveBeenCalled(); + store.setSession('123'); + expect(manager.setSession).toHaveBeenCalled(); + expect(localStorage.getItem('my-key')).toBe('"123"'); + await expect(store.getSession({ optional: true })).resolves.toBe('123'); + }); + it('should forward sessionState calls', () => { const manager = new MockManager(); const store = new AuthSessionStore({ manager, ...defaultOptions }); diff --git a/packages/core-app-api/src/lib/AuthSessionManager/AuthSessionStore.ts b/packages/core-app-api/src/lib/AuthSessionManager/AuthSessionStore.ts index 057a70e58d..fc15da7af8 100644 --- a/packages/core-app-api/src/lib/AuthSessionManager/AuthSessionStore.ts +++ b/packages/core-app-api/src/lib/AuthSessionManager/AuthSessionStore.ts @@ -15,7 +15,6 @@ */ import { - SessionManager, MutableSessionManager, SessionScopesFunc, SessionShouldRefreshFunc, @@ -40,7 +39,7 @@ type Options = { * * Session is serialized to JSON with special support for following types: Set. */ -export class AuthSessionStore implements SessionManager { +export class AuthSessionStore implements MutableSessionManager { private readonly manager: MutableSessionManager; private readonly storageKey: string; private readonly sessionShouldRefreshFunc: SessionShouldRefreshFunc; @@ -63,6 +62,11 @@ export class AuthSessionStore implements SessionManager { }); } + setSession(session: T | undefined): void { + this.manager.setSession(session); + this.saveSession(session); + } + async getSession(options: GetSessionOptions): Promise { const { scopes } = options; const session = this.loadSession(); diff --git a/packages/core-app-api/src/lib/AuthSessionManager/OptionalRefreshSessionManagerMux.test.ts b/packages/core-app-api/src/lib/AuthSessionManager/OptionalRefreshSessionManagerMux.test.ts new file mode 100644 index 0000000000..919948f531 --- /dev/null +++ b/packages/core-app-api/src/lib/AuthSessionManager/OptionalRefreshSessionManagerMux.test.ts @@ -0,0 +1,138 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Observable, SessionState } from '@backstage/core-plugin-api'; +import { OptionalRefreshSessionManagerMux } from './OptionalRefreshSessionManagerMux'; +import { MutableSessionManager, SessionManager } from './types'; + +class MockManager implements MutableSessionManager { + constructor(public session?: string) {} + + setSession(session: string | undefined): void { + this.session = session; + } + async getSession(): Promise { + return this.session; + } + async removeSession(): Promise { + delete this.session; + } + sessionState$(): Observable { + throw new Error('Method not implemented.'); + } +} + +function trackState(manager: SessionManager) { + const states = new Array(); + manager + .sessionState$() + .subscribe(state => states.push(state === SessionState.SignedIn)); + return states; +} + +describe('OptionalRefreshSessionManagerMux', () => { + it('finds no session', async () => { + const mux = new OptionalRefreshSessionManagerMux({ + staticSessionManager: new MockManager(), + refreshingSessionManager: new MockManager(), + sessionCanRefresh: () => false, + }); + + const states = trackState(mux); + await expect(mux.getSession({})).resolves.toBe(undefined); + expect(states).toEqual([false]); + }); + + it('prioritizes a static session', async () => { + const mux = new OptionalRefreshSessionManagerMux({ + staticSessionManager: new MockManager('static'), + refreshingSessionManager: new MockManager('refreshing'), + sessionCanRefresh: () => false, + }); + + const states = trackState(mux); + await expect(mux.getSession({})).resolves.toBe('static'); + expect(states).toEqual([false, true]); + }); + + it('transfers a refreshing session to the static manager', async () => { + const staticSessionManager = new MockManager(); + const refreshingSessionManager = new MockManager('refreshing'); + const mux = new OptionalRefreshSessionManagerMux({ + staticSessionManager, + refreshingSessionManager, + sessionCanRefresh: () => false, + }); + + const states = trackState(mux); + expect(staticSessionManager.session).toBeUndefined(); + await expect(mux.getSession({})).resolves.toBe('refreshing'); + expect(staticSessionManager.session).toBe('refreshing'); + expect(states).toEqual([false, true]); + }); + + it('relies on the refreshing manager if refresh is available', async () => { + const staticSessionManager = new MockManager(); + const refreshingSessionManager = new MockManager('refreshing'); + const mux = new OptionalRefreshSessionManagerMux({ + staticSessionManager, + refreshingSessionManager, + sessionCanRefresh: () => true, + }); + + const states = trackState(mux); + await expect(mux.getSession({})).resolves.toBe('refreshing'); + expect(staticSessionManager.session).toBeUndefined(); + expect(states).toEqual([false, true]); + }); + + it('can switch between refreshing and static sessions', async () => { + let canRefresh = true; + const staticSessionManager = new MockManager(); + const refreshingSessionManager = new MockManager('refreshing'); + const mux = new OptionalRefreshSessionManagerMux({ + staticSessionManager, + refreshingSessionManager, + sessionCanRefresh: () => canRefresh, + }); + + const states = trackState(mux); + await expect(mux.getSession({})).resolves.toBe('refreshing'); + expect(staticSessionManager.session).toBeUndefined(); + canRefresh = false; + await expect(mux.getSession({})).resolves.toBe('refreshing'); + expect(staticSessionManager.session).toBe('refreshing'); + + expect(states).toEqual([false, true]); + }); + + it('removes sessions from both managers', async () => { + const staticSessionManager = new MockManager('static'); + const refreshingSessionManager = new MockManager('refreshing'); + const mux = new OptionalRefreshSessionManagerMux({ + staticSessionManager, + refreshingSessionManager, + sessionCanRefresh: () => true, + }); + + const states = trackState(mux); + await expect(mux.getSession({})).resolves.toBe('static'); + await mux.removeSession(); + expect(staticSessionManager.session).toBeUndefined(); + expect(refreshingSessionManager.session).toBeUndefined(); + expect(states).toEqual([false, true, false]); + }); +}); diff --git a/packages/core-app-api/src/lib/AuthSessionManager/OptionalRefreshSessionManagerMux.ts b/packages/core-app-api/src/lib/AuthSessionManager/OptionalRefreshSessionManagerMux.ts new file mode 100644 index 0000000000..221a7635ea --- /dev/null +++ b/packages/core-app-api/src/lib/AuthSessionManager/OptionalRefreshSessionManagerMux.ts @@ -0,0 +1,105 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Observable, SessionState } from '@backstage/core-plugin-api'; +import { + SessionManager, + MutableSessionManager, + GetSessionOptions, +} from './types'; +import { SessionStateTracker } from './SessionStateTracker'; + +type Options = { + /** + * A callback that is called to determine whether a given session supports refresh + */ + sessionCanRefresh: (session: T) => boolean; + + /** + * The session manager that is used if the a session does not support refresh. + */ + staticSessionManager: MutableSessionManager; + + /** + * The session manager that is used if the a session supports refresh. + */ + refreshingSessionManager: SessionManager; +}; + +/** + * OptionalRefreshSessionManagerMux wraps two different session managers, one for + * static session storage and another one that supports refresh. For each session + * that is retrieved is checked for whether it supports refresh. If it does, the + * refreshing session manager is used, otherwise the static session manager is used. + */ +export class OptionalRefreshSessionManagerMux implements SessionManager { + private readonly stateTracker = new SessionStateTracker(); + + private readonly sessionCanRefresh: (session: T) => boolean; + private readonly staticSessionManager: MutableSessionManager; + private readonly refreshingSessionManager: SessionManager; + + constructor(options: Options) { + this.sessionCanRefresh = options.sessionCanRefresh; + this.staticSessionManager = options.staticSessionManager; + this.refreshingSessionManager = options.refreshingSessionManager; + } + + async getSession(options: GetSessionOptions): Promise { + // First we check if there is an existing static session, using an optional request + const staticSession = await this.staticSessionManager.getSession({ + ...options, + optional: true, + }); + if (staticSession) { + this.stateTracker.setIsSignedIn(true); + return staticSession; + } + + // If there is no static session available, we ask the refresh manager to get a session + const session = await this.refreshingSessionManager.getSession(options); + + // Handling the case where the session request is optional + if (!session) { + this.stateTracker.setIsSignedIn(false); + return undefined; + } + + // Next we check if the session we received from the refreshing manager can actually + // be refreshed. If it can, we use this session without storing it in the static manager. + if (this.sessionCanRefresh(session)) { + this.stateTracker.setIsSignedIn(true); + return session; + } + + // If the session can't be refreshed, we store it in the static manager + this.staticSessionManager.setSession(session); + this.stateTracker.setIsSignedIn(true); + return session; + } + + async removeSession(): Promise { + await Promise.all([ + this.refreshingSessionManager.removeSession(), + this.staticSessionManager.removeSession(), + ]); + this.stateTracker.setIsSignedIn(false); + } + + sessionState$(): Observable { + return this.stateTracker.sessionState$(); + } +} diff --git a/packages/core-components/CHANGELOG.md b/packages/core-components/CHANGELOG.md index 0241234574..1b56b033b5 100644 --- a/packages/core-components/CHANGELOG.md +++ b/packages/core-components/CHANGELOG.md @@ -1,5 +1,24 @@ # @backstage/core-components +## 0.3.3 + +### Patch Changes + +- d041655a7: Fix accessibility issue in ``. The component doesn't render anymore an hidden `textarea` containing the text to be copied. +- 6d76bca85: Handle changes to nodes passed into `` correctly. +- Updated dependencies + - @backstage/config@0.1.8 + +## 0.3.2 + +### Patch Changes + +- a3f3cff3b: Change the default hover experience for the sidebar to be not jumpy & add visual separation between sidebar & Entity Page tabs for dark mode. +- 6b1afe8c0: Add a configurable `palette.bursts.gradient` property to the Backstage theme, to support customizing the gradients in the `ItemCard` header. +- Updated dependencies + - @backstage/config@0.1.7 + - @backstage/theme@0.2.10 + ## 0.3.1 ### Patch Changes diff --git a/packages/core-components/api-report.md b/packages/core-components/api-report.md index 42404fb0c0..f73eee1f13 100644 --- a/packages/core-components/api-report.md +++ b/packages/core-components/api-report.md @@ -15,7 +15,6 @@ import { Column } from '@material-table/core'; import { CommonProps } from '@material-ui/core/OverridableComponent'; import { ComponentClass } from 'react'; import { ComponentProps } from 'react'; -import { ComponentType } from 'react'; import { Context } from 'react'; import { default as CSS_2 } from 'csstype'; import { CSSProperties } from 'react'; @@ -28,6 +27,7 @@ import { LinkProps as LinkProps_2 } from '@material-ui/core'; import { LinkProps as LinkProps_3 } from 'react-router-dom'; import { MaterialTableProps } from '@material-table/core'; import { NavLinkProps } from 'react-router-dom'; +import { Palette } from '@material-ui/core/styles/createPalette'; import { ProfileInfoApi } from '@backstage/core-plugin-api'; import { PropsWithChildren } from 'react'; import PropTypes from 'prop-types'; diff --git a/packages/core-components/package.json b/packages/core-components/package.json index 69f2019943..ae6daece4d 100644 --- a/packages/core-components/package.json +++ b/packages/core-components/package.json @@ -1,7 +1,7 @@ { "name": "@backstage/core-components", "description": "Core components used by Backstage plugins and apps", - "version": "0.3.1", + "version": "0.3.3", "private": false, "publishConfig": { "access": "public", @@ -29,10 +29,10 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@backstage/config": "^0.1.6", + "@backstage/config": "^0.1.8", "@backstage/core-plugin-api": "^0.1.6", "@backstage/errors": "^0.1.1", - "@backstage/theme": "^0.2.9", + "@backstage/theme": "^0.2.10", "@material-table/core": "^3.1.0", "@material-ui/core": "^4.12.2", "@material-ui/icons": "^4.9.1", @@ -58,7 +58,7 @@ "react": "^16.12.0", "react-dom": "^16.12.0", "react-helmet": "6.1.0", - "react-hook-form": "^6.15.4", + "react-hook-form": "^7.12.2", "react-markdown": "^5.0.2", "react-router": "6.0.0-beta.0", "react-router-dom": "6.0.0-beta.0", @@ -70,8 +70,8 @@ "zen-observable": "^0.8.15" }, "devDependencies": { - "@backstage/core-app-api": "^0.1.8", - "@backstage/cli": "^0.7.8", + "@backstage/core-app-api": "^0.1.10", + "@backstage/cli": "^0.7.9", "@backstage/test-utils": "^0.1.17", "@testing-library/jest-dom": "^5.10.1", "@testing-library/react": "^11.2.5", diff --git a/packages/core-components/src/components/CodeSnippet/CodeSnippet.test.tsx b/packages/core-components/src/components/CodeSnippet/CodeSnippet.test.tsx index 298a8d5780..a4ffd3462b 100644 --- a/packages/core-components/src/components/CodeSnippet/CodeSnippet.test.tsx +++ b/packages/core-components/src/components/CodeSnippet/CodeSnippet.test.tsx @@ -15,8 +15,6 @@ */ import React from 'react'; -import { fireEvent } from '@testing-library/react'; -import { act } from 'react-dom/test-utils'; import { renderInTestApp } from '@backstage/test-utils'; import { CodeSnippet } from './CodeSnippet'; @@ -56,19 +54,4 @@ describe('', () => { expect(getByText('2')).toBeInTheDocument(); expect(getByText('3')).toBeInTheDocument(); }); - - it('copy code using button', async () => { - jest.useFakeTimers(); - document.execCommand = jest.fn(); - const { getByTitle } = await renderInTestApp( - , - ); - const button = getByTitle('Text copied to clipboard'); - fireEvent.click(button); - act(() => { - jest.runAllTimers(); - }); - expect(document.execCommand).toHaveBeenCalled(); - jest.useRealTimers(); - }); }); diff --git a/packages/core-components/src/components/CopyTextButton/CopyTextButton.test.tsx b/packages/core-components/src/components/CopyTextButton/CopyTextButton.test.tsx index acd0fa809c..b84d3919e8 100644 --- a/packages/core-components/src/components/CopyTextButton/CopyTextButton.test.tsx +++ b/packages/core-components/src/components/CopyTextButton/CopyTextButton.test.tsx @@ -19,8 +19,9 @@ import { fireEvent } from '@testing-library/react'; import { act } from 'react-dom/test-utils'; import { renderInTestApp } from '@backstage/test-utils'; import { CopyTextButton } from './CopyTextButton'; -import { ApiRegistry, ApiProvider } from '@backstage/core-app-api'; +import { ApiProvider, ApiRegistry } from '@backstage/core-app-api'; import { errorApiRef, ErrorApi } from '@backstage/core-plugin-api'; +import { useCopyToClipboard } from 'react-use'; jest.mock('popper.js', () => { const PopperJS = jest.requireActual('popper.js'); @@ -33,6 +34,17 @@ jest.mock('popper.js', () => { }; }); +jest.mock('react-use', () => { + const original = jest.requireActual('react-use'); + + return { + ...original, + useCopyToClipboard: jest + .fn() + .mockImplementation(original.useCopyToClipboard), + }; +}); + const props = { text: 'mockText', tooltipDelay: 2, @@ -43,9 +55,7 @@ const apiRegistry = ApiRegistry.from([ [ errorApiRef, { - post(error) { - throw error; - }, + post: jest.fn(), error$: jest.fn(), } as ErrorApi, ], @@ -53,17 +63,22 @@ const apiRegistry = ApiRegistry.from([ describe('', () => { it('renders without exploding', async () => { - const { getByDisplayValue } = await renderInTestApp( + const { getByTitle, queryByText } = await renderInTestApp( , ); - getByDisplayValue('mockText'); + expect(getByTitle('mockTooltip')).toBeInTheDocument(); + expect(queryByText('mockTooltip')).not.toBeInTheDocument(); }); - it('displays tooltip on click', async () => { + it('displays tooltip and copy the text on click', async () => { jest.useFakeTimers(); - document.execCommand = jest.fn(); + + const spy = useCopyToClipboard as jest.Mock; + const copy = jest.fn(); + spy.mockReturnValue([{}, copy]); + const rendered = await renderInTestApp( @@ -74,8 +89,22 @@ describe('', () => { act(() => { jest.runAllTimers(); }); - expect(document.execCommand).toHaveBeenCalled(); + expect(copy).toHaveBeenCalledWith('mockText'); rendered.getByText('mockTooltip'); jest.useRealTimers(); }); + + it('reports copy errors', async () => { + const spy = useCopyToClipboard as jest.Mock; + + const error = new Error('just an error'); + spy.mockReturnValue([{ error }, jest.fn()]); + + await renderInTestApp( + + + , + ); + expect(apiRegistry.get(errorApiRef)?.post).toHaveBeenCalledWith(error); + }); }); diff --git a/packages/core-components/src/components/CopyTextButton/CopyTextButton.tsx b/packages/core-components/src/components/CopyTextButton/CopyTextButton.tsx index b5bc516d1a..cc0ecaf76f 100644 --- a/packages/core-components/src/components/CopyTextButton/CopyTextButton.tsx +++ b/packages/core-components/src/components/CopyTextButton/CopyTextButton.tsx @@ -18,7 +18,8 @@ import { errorApiRef, useApi } from '@backstage/core-plugin-api'; import { IconButton, Tooltip } from '@material-ui/core'; import CopyIcon from '@material-ui/icons/FileCopy'; import PropTypes from 'prop-types'; -import React, { MouseEventHandler, useRef, useState } from 'react'; +import React, { MouseEventHandler, useEffect, useState } from 'react'; +import { useCopyToClipboard } from 'react-use'; /** * Copy text button with visual feedback in the form of @@ -52,30 +53,23 @@ export const CopyTextButton = (props: Props) => { ...props, }; const errorApi = useApi(errorApiRef); - const inputRef = useRef(null); const [open, setOpen] = useState(false); + const [{ error }, copyToClipboard] = useCopyToClipboard(); + + useEffect(() => { + if (error) { + errorApi.post(error); + } + }, [error, errorApi]); const handleCopyClick: MouseEventHandler = e => { e.stopPropagation(); setOpen(true); - - try { - if (inputRef.current) { - inputRef.current.select(); - document.execCommand('copy'); - } - } catch (error) { - errorApi.post(error); - } + copyToClipboard(text); }; return ( <> -