diff --git a/.changeset/angry-hounds-bow.md b/.changeset/angry-hounds-bow.md new file mode 100644 index 0000000000..4bbfffd4f1 --- /dev/null +++ b/.changeset/angry-hounds-bow.md @@ -0,0 +1,5 @@ +--- +'@backstage/integration': patch +--- + +Do not throw in `ScmIntegration` `byUrl` for invalid URLs diff --git a/.changeset/beige-ladybugs-sip.md b/.changeset/beige-ladybugs-sip.md new file mode 100644 index 0000000000..f1ef84e233 --- /dev/null +++ b/.changeset/beige-ladybugs-sip.md @@ -0,0 +1,5 @@ +--- +'@backstage/core-components': patch +--- + +Improve UX of the Sidebar by adding SidebarScrollWrapper component allowing the user to scroll through Plugins & Shortcuts on smaller screens. Prevent the Sidebar from opening on click on small devices diff --git a/.changeset/breezy-kings-applaud.md b/.changeset/breezy-kings-applaud.md new file mode 100644 index 0000000000..867d135739 --- /dev/null +++ b/.changeset/breezy-kings-applaud.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-pagerduty': patch +--- + +Update README diff --git a/.changeset/brown-olives-hang.md b/.changeset/brown-olives-hang.md new file mode 100644 index 0000000000..64ce4e25d5 --- /dev/null +++ b/.changeset/brown-olives-hang.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-scaffolder-backend': patch +--- + +add default branch property for publish GitLab, Bitbucket and Azure actions diff --git a/.changeset/eight-pots-remember.md b/.changeset/eight-pots-remember.md new file mode 100644 index 0000000000..34c6d018b5 --- /dev/null +++ b/.changeset/eight-pots-remember.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-scaffolder-backend': patch +--- + +Added filesystem remove/rename built-in actions diff --git a/.changeset/eighty-deers-jam.md b/.changeset/eighty-deers-jam.md new file mode 100644 index 0000000000..df8b84aff1 --- /dev/null +++ b/.changeset/eighty-deers-jam.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-import': patch +--- + +More helpful error message when trying to import by folder from non-github diff --git a/.changeset/fair-falcons-scream.md b/.changeset/fair-falcons-scream.md new file mode 100644 index 0000000000..1f2f095c81 --- /dev/null +++ b/.changeset/fair-falcons-scream.md @@ -0,0 +1,7 @@ +--- +'@backstage/core-components': patch +'@backstage/plugin-explore': patch +--- + +- Enhanced core `Button` component to open external links in new tab. +- Replaced the use of `Button` component from material by `core-components` in tools card. diff --git a/.changeset/flat-donkeys-rhyme.md b/.changeset/flat-donkeys-rhyme.md new file mode 100644 index 0000000000..c98f4aedc5 --- /dev/null +++ b/.changeset/flat-donkeys-rhyme.md @@ -0,0 +1,5 @@ +--- +'@backstage/catalog-model': minor +--- + +Changed the regex to validate names following the Kubernetes validation rule, this allow to be more permissive validating the name of the object in Backstage. diff --git a/.changeset/gorgeous-timers-cover.md b/.changeset/gorgeous-timers-cover.md new file mode 100644 index 0000000000..de71d740c0 --- /dev/null +++ b/.changeset/gorgeous-timers-cover.md @@ -0,0 +1,5 @@ +--- +'@backstage/techdocs-common': patch +--- + +Fix openStack swift publisher encoding issue. Remove utf8 forced encoding on binary files diff --git a/.changeset/happy-pugs-notice.md b/.changeset/happy-pugs-notice.md new file mode 100644 index 0000000000..4120018da9 --- /dev/null +++ b/.changeset/happy-pugs-notice.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog': patch +--- + +Change catalog page layout to use Grid components to improve responsiveness diff --git a/.changeset/honest-cars-glow.md b/.changeset/honest-cars-glow.md new file mode 100644 index 0000000000..3fdc6e335a --- /dev/null +++ b/.changeset/honest-cars-glow.md @@ -0,0 +1,54 @@ +--- +'@backstage/backend-common': patch +--- + +Added a `readUrl` method to the `UrlReader` interface that allows for complex response objects and is intended to replace the `read` method. This new method is currently optional to implement which allows for a soft migration to `readUrl` instead of `read` in the future. + +The main use case for `readUrl` returning an object instead of solely a read buffer is to allow for additional metadata such as ETag, which is a requirement for more efficient catalog processing. + +The `GithubUrlReader` and `GitlabUrlReader` readers fully implement `readUrl`. The other existing readers implement the new method but do not propagate or return ETags. + +While the `readUrl` method is not yet required, it will be in the future, and we already log deprecation warnings when custom `UrlReader` implementations that do not implement `readUrl` are used. We therefore recommend that any existing custom implementations are migrated to implement `readUrl`. + +The old `read` and the new `readUrl` methods can easily be implemented using one another, but we recommend moving the chunk of the implementation to the new `readUrl` method as `read` is being removed, for example this: + +```ts +class CustomUrlReader implements UrlReader { + read(url: string): Promise { + const res = await fetch(url); + + if (!res.ok) { + // error handling ... + } + + return Buffer.from(await res.text()); + } +} +``` + +Can be migrated to something like this: + +```ts +class CustomUrlReader implements UrlReader { + read(url: string): Promise { + const res = await this.readUrl(url); + return res.buffer(); + } + + async readUrl( + url: string, + _options?: ReadUrlOptions, + ): Promise { + const res = await fetch(url); + + if (!res.ok) { + // error handling ... + } + + const buffer = Buffer.from(await res.text()); + return { buffer: async () => buffer }; + } +} +``` + +While there is no usage of the ETag capability yet in the main Backstage packages, you can already add it to your custom implementations. To do so, refer to the documentation of the `readUrl` method and surrounding types, and the existing implementation in `packages/backend-common/src/reading/GithubUrlReader.ts`. diff --git a/.changeset/large-mice-sin.md b/.changeset/large-mice-sin.md new file mode 100644 index 0000000000..77aba8d3c5 --- /dev/null +++ b/.changeset/large-mice-sin.md @@ -0,0 +1,6 @@ +--- +'@backstage/plugin-catalog-backend-module-ldap': minor +--- + +Add extension points to the `LdapOrgReaderProcessor` to make it possible to do more advanced modifications +of the ingested users and groups. diff --git a/.changeset/late-pants-itch.md b/.changeset/late-pants-itch.md new file mode 100644 index 0000000000..7bd3087316 --- /dev/null +++ b/.changeset/late-pants-itch.md @@ -0,0 +1,5 @@ +--- +'@backstage/core-components': patch +--- + +Show scroll bar of the sidebar wrapper only on hover diff --git a/.changeset/rotten-cooks-try.md b/.changeset/rotten-cooks-try.md new file mode 100644 index 0000000000..ccbd315ebe --- /dev/null +++ b/.changeset/rotten-cooks-try.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-ilert': patch +--- + +chore: bump `@date-io/luxon` from 1.3.13 to 2.10.11 diff --git a/.changeset/rotten-walls-cross.md b/.changeset/rotten-walls-cross.md new file mode 100644 index 0000000000..498c10b203 --- /dev/null +++ b/.changeset/rotten-walls-cross.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +Make use of the new `readUrl` method on `UrlReader` from `@backstage/backend-common`. diff --git a/.changeset/serious-falcons-greet.md b/.changeset/serious-falcons-greet.md new file mode 100644 index 0000000000..f793ed26d4 --- /dev/null +++ b/.changeset/serious-falcons-greet.md @@ -0,0 +1,5 @@ +--- +'@backstage/integration': patch +--- + +Fix downloads from repositories located at bitbucket.org diff --git a/.changeset/shy-garlics-tap.md b/.changeset/shy-garlics-tap.md new file mode 100644 index 0000000000..58c660e864 --- /dev/null +++ b/.changeset/shy-garlics-tap.md @@ -0,0 +1,5 @@ +--- +'@backstage/create-app': patch +--- + +Use SidebarScrollWrapper to improve responsiveness of the current sidebar. Change: Wrap a section of SidebarItems with this component to enable scroll for smaller screens. It can also be used in sidebar plugins (see shortcuts plugin for an example). diff --git a/.changeset/silent-horses-fry.md b/.changeset/silent-horses-fry.md new file mode 100644 index 0000000000..2b7c562607 --- /dev/null +++ b/.changeset/silent-horses-fry.md @@ -0,0 +1,5 @@ +--- +'@backstage/core-components': patch +--- + +Fix error in error panel, and console warnings about DOM nesting pre inside p diff --git a/.changeset/six-birds-approve.md b/.changeset/six-birds-approve.md new file mode 100644 index 0000000000..ce666dd09c --- /dev/null +++ b/.changeset/six-birds-approve.md @@ -0,0 +1,6 @@ +--- +'@backstage/core-components': patch +'@backstage/plugin-org': patch +--- + +Add edit button to Group Profile Card diff --git a/.changeset/tasty-chairs-flash.md b/.changeset/tasty-chairs-flash.md new file mode 100644 index 0000000000..8be48d92b3 --- /dev/null +++ b/.changeset/tasty-chairs-flash.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-scaffolder': patch +--- + +Pass through the `idToken` in `Authorization` Header for `listActions` request diff --git a/.changeset/violet-maps-smell.md b/.changeset/violet-maps-smell.md new file mode 100644 index 0000000000..fdb4c6b46d --- /dev/null +++ b/.changeset/violet-maps-smell.md @@ -0,0 +1,6 @@ +--- +'@backstage/backend-common': patch +'@backstage/plugin-scaffolder-backend': patch +--- + +add defaultBranch property for publish GitHub action diff --git a/.changeset/wet-queens-deny.md b/.changeset/wet-queens-deny.md new file mode 100644 index 0000000000..1e446c619c --- /dev/null +++ b/.changeset/wet-queens-deny.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-scaffolder-backend': patch +--- + +removing mandatory of protection for the default branch, that could be handled by the GitHub automation in async manner, thus throwing floating errors diff --git a/.changeset/young-pigs-beg.md b/.changeset/young-pigs-beg.md new file mode 100644 index 0000000000..bfdf726911 --- /dev/null +++ b/.changeset/young-pigs-beg.md @@ -0,0 +1,5 @@ +--- +'@backstage/core-components': patch +--- + +Add custom styles to scroll bar of the sidebar wrapper to fix flaky behaviour diff --git a/.github/styles/vocab.txt b/.github/styles/vocab.txt index 846e050c49..f6826cbefe 100644 --- a/.github/styles/vocab.txt +++ b/.github/styles/vocab.txt @@ -137,6 +137,7 @@ maintainership makefile md memcache +memoized microservice microservices microsite @@ -281,6 +282,7 @@ unregistration untracked upvote url +URLs utils validator validators diff --git a/ADOPTERS.md b/ADOPTERS.md index 83fbc79eca..1dcd4a774f 100644 --- a/ADOPTERS.md +++ b/ADOPTERS.md @@ -36,3 +36,4 @@ | [HelloFresh](https://www.hellofresh.de/) | [@iammuho](https://github.com/iammuho), [@ElenaForester](https://github.com/ElenaForester), [@diegomarangoni](https://github.com/diegomarangoni) | Our developer portal at HelloFresh - Spread across an organisation of 500+ engineers globally. | | [FactSet](https://www.factset.com/) | [@kuangp](https://github.com/kuangp) | Developer portal to provide discoverability to all internal components, APIs, documentation, and scaffold templates with integrations to our internal infrastructure tools. | | [Workrise](https://www.workrise.com/) | [Michael Rode](https://github.com/michaelrode) | Developer portal, main gateway to our infrastructure, documentation and internal tooling. | +| [RedVentures](https://www.redventures.com/) | [Chris Diaz](https://github.com/codingdiaz) | Developer portal that brings everything an engineer needs to provide value into a single pane of glass. | diff --git a/README.md b/README.md index f8b54ad200..895106c967 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,8 @@ # [Backstage](https://backstage.io) +_During the month of July the majority of the maintainers will be on summer vacation 🏖️ Development will continue as usual, but expect a slower pace for discussions and PR reviews. Why not take this opportunity to [build a plugin](https://backstage.io/docs/plugins/)?_ + [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) [![CNCF Status](https://img.shields.io/badge/cncf%20status-sandbox-blue.svg)](https://www.cncf.io/projects) [![Main CI Build](https://github.com/backstage/backstage/workflows/Main%20Master%20Build/badge.svg)](https://github.com/backstage/backstage/actions?query=workflow%3A%22Main+Master+Build%22) diff --git a/cypress/src/integration/catalog.ts b/cypress/src/integration/catalog.ts index 1db2eb8a13..20dcdd9b8a 100644 --- a/cypress/src/integration/catalog.ts +++ b/cypress/src/integration/catalog.ts @@ -23,7 +23,7 @@ describe('Catalog', () => { cy.visit('/catalog'); - cy.contains('Owned (8)').should('be.visible'); + cy.contains('Owned (10)').should('be.visible'); }); }); }); diff --git a/cypress/src/integration/integrations.ts b/cypress/src/integration/integrations.ts index 3e3a10aca2..4703c18881 100644 --- a/cypress/src/integration/integrations.ts +++ b/cypress/src/integration/integrations.ts @@ -27,8 +27,10 @@ describe('Integrations', () => { type: 'url', }); + cy.wait(5000); + cy.visit('/catalog'); - cy.contains('All').click(); + cy.get('[data-testid="user-picker-all"]').click(); cy.get('table').should('contain', 'github-repo'); cy.get('table').should('contain', 'github-repo-nested'); }); @@ -52,8 +54,10 @@ describe('Integrations', () => { type: 'url', }); + cy.wait(5000); + cy.visit('/catalog'); - cy.contains('All').click(); + cy.get('[data-testid="user-picker-all"]').click(); cy.get('table').should('contain', 'gitlab-repo'); cy.get('table').should('contain', 'gitlab-repo-nested'); }); @@ -67,8 +71,10 @@ describe('Integrations', () => { type: 'url', }); + cy.wait(5000); + cy.visit('/catalog'); - cy.contains('All').click(); + cy.get('[data-testid="user-picker-all"]').click(); cy.get('table').should('contain', 'bitbucket-repo'); cy.get('table').should('contain', 'bitbucket-repo-nested'); }); diff --git a/docs/features/techdocs/troubleshooting.md b/docs/features/techdocs/troubleshooting.md index e6efefc578..d8b413be5c 100644 --- a/docs/features/techdocs/troubleshooting.md +++ b/docs/features/techdocs/troubleshooting.md @@ -55,3 +55,20 @@ INFO - Start watching changes [I 210115 19:00:45 handlers:64] Start detecting changes INFO - Start detecting changes ``` + +## PlantUML with `svg_object` doesn't render + +The [plantuml-markdown](https://pypi.org/project/plantuml-markdown/) MkDocs +plugin available in +[`mkdocs-techdocs-core`](https://github.com/backstage/mkdocs-techdocs-core) +supports different formats for rendering diagrams. TechDocs does however not +support all of them. + +The `svg_object` format renders a diagram as an HTML `` tag but this is +not allowed as it enables bad actors to inject malicious content into +documentation pages. See +[CVE-2021-32661](https://github.com/advisories/GHSA-gg96-f8wr-p89f) for more +details. + +Instead use `svg_inline` which renders as an `` tag and provides the same +benefits as `svg_object`. diff --git a/docs/integrations/ldap/org.md b/docs/integrations/ldap/org.md index 836601971c..d1438aba8b 100644 --- a/docs/integrations/ldap/org.md +++ b/docs/integrations/ldap/org.md @@ -122,8 +122,8 @@ The DN under which users are stored, e.g. #### users.options The search options to use when sending the query to the server, when reading all -users. All of the options are shown below, with their default values, but they -are all optional. +users. All the options are shown below, with their default values, but they are +all optional. ```yaml options: @@ -158,8 +158,8 @@ set: Mappings from well known entity fields, to LDAP attribute names. This is where you are able to define how to interpret the attributes of each LDAP result item, -and to move them into the corresponding entity fields. All of the options are -shown below, with their default values, but they are all optional. +and to move them into the corresponding entity fields. All the options are shown +below, with their default values, but they are all optional. If you leave out an optional mapping, it will still be copied using that default value. For example, even if you do not put in the field `displayName` in your @@ -204,8 +204,8 @@ The DN under which groups are stored, e.g. #### groups.options The search options to use when sending the query to the server, when reading all -groups. All of the options are shown below, with their default values, but they -are all optional. +groups. All the options are shown below, with their default values, but they are +all optional. ```yaml options: @@ -282,3 +282,35 @@ map: # the spec.children field of the entity. members: member ``` + +## Customize the Processor + +In case you want to customize the ingested entities, the +`LdapOrgReaderProcessor` allows to pass transformers for users and groups. + +1. Create a transformer: + +```ts +export async function myGroupTransformer( + vendor: LdapVendor, + config: GroupConfig, + group: SearchEntry, +): Promise { + // Transformations may change namespace, change entity naming pattern, fill + // profile with more or other details... + + // Create the group entity on your own, or wrap the default transformer + return await defaultGroupTransformer(vendor, config, group); +} +``` + +2. Configure the processor with the transformer: + +```ts +builder.addProcessor( + LdapOrgReaderProcessor.fromConfig(config, { + logger, + groupTransformer: myGroupTransformer, + }), +); +``` diff --git a/docs/plugins/url-reader.md b/docs/plugins/url-reader.md new file mode 100644 index 0000000000..41aba98a66 --- /dev/null +++ b/docs/plugins/url-reader.md @@ -0,0 +1,274 @@ +--- +id: url-reader +title: URL Reader +sidebar_label: URL Reader +# prettier-ignore +description: URL Reader is a backend core API responsible for reading files from external locations. +--- + +## Concept + +Some of the core plugins of Backstage have to read files from an external +location. [Software Catalog](../features/software-catalog/index.md) has to read +the [`catalog-info.yaml`](../features/software-catalog/descriptor-format.md) +entity descriptor files to register and track an entity. +[Software Templates](../features/software-templates/index.md) have to download +the template skeleton files before creating a new component. +[TechDocs](../features/techdocs/README.md) has to download the markdown source +files before generating a documentation site. + +Since, the requirement for reading files is so essential for Backstage plugins, +the +[`@backstage/backend-common`](https://github.com/backstage/backstage/tree/master/packages/backend-common) +package provides a dedicated API for reading from such URL based remote +locations like GitHub, GitLab, Bitbucket, Google Cloud Storage, etc. This is +commonly referred to as "URL Reader". It takes care of making authenticated +requests to the remote host so that private files can be read securely. If users +have [GitHub App based authentication](github-apps.md) set up, URL Reader even +refreshes the token, to avoid reaching the GitHub API rate limit. + +As a result, plugin authors do not have to worry about any of these problems +when trying to read files. + +## Interface + +When the Backstage backend starts, a new instance of URL Reader is created. You +can see this in the index file of your Backstage backend +i.e.`packages/backend/src/index.ts`. +[Example](https://github.com/backstage/backstage/blob/ebbe91dbe79038a61d35cf6ed2d96e0e0d5a15f3/packages/backend/src/index.ts#L57) + +```ts +// File: packages/backend/src/index.ts + +import { URLReaders } from '@backstage/backend-common'; + +function makeCreateEnv(config: Config) { + // .... + const reader = UrlReaders.default({ logger, config }); + // +} +``` + +This instance contains all +[the default URL Reader providers](https://github.com/backstage/backstage/blob/master/packages/backend-common/src/reading/UrlReaders.ts) +in the backend-common package including GitHub, GitLab, Bitbucket, Azure, Google +GCS. As the need arises, more URL Readers are being written to support different +providers. + +The generic interface of a URL Reader instance looks like this. + +```ts +export type UrlReader = { + /* Used to read a single file and return its content. */ + read(url: string): Promise; + /** + * A replacement for the read method that supports options and complex responses. + * + * Use this whenever it is available, as the read method will be deprecated and + * eventually removed in the future. + */ + readUrl?(url: string, options?: ReadUrlOptions): Promise; + + /* Used to read a file tree and download as a directory. */ + readTree(url: string, options?: ReadTreeOptions): Promise; + /* Used to search a file in a tree. */ + search(url: string, options?: SearchOptions): Promise; +}; +``` + +## Using a URL Reader inside a plugin + +The `reader` instance is available in the backend plugin environment and passed +on to all the backend plugins. You can see an +[example](https://github.com/backstage/backstage/blob/b0be185369ebaad22255b7cdf18535d1d4ffd0e7/packages/backend/src/plugins/techdocs.ts#L31). +When any of the methods on this instance is called with a URL, URL Reader +extracts the host for that URL (e.g. `github.com`, `ghe.mycompany.com`, etc.). +Using the +[`@backstage/integration`](https://github.com/backstage/backstage/tree/master/packages/integration) +package, it looks inside the +[`integrations:`](https://github.com/backstage/backstage/blob/d5c83bb889b8142e343ebc4e4c0b90a02d1c1a3d/app-config.yaml#L134-L158) +config of the `app-config.yaml` to find out how to work with the host based on +the configs provided like authentication token, API base URL, etc. + +Make sure your plugin-specific backend file at +`packages/backend/src/plugins/.ts` is forwarding the `reader` instance +passed on as the `PluginEnvironment` to the actual plugin's `createRouter` +function. See how this is done in +[Catalog](https://github.com/backstage/backstage/blob/d5c83bb889b8142e343ebc4e4c0b90a02d1c1a3d/packages/backend/src/plugins/catalog.ts#L25-L27) +and +[TechDocs](https://github.com/backstage/backstage/blob/d5c83bb889b8142e343ebc4e4c0b90a02d1c1a3d/packages/backend/src/plugins/techdocs.ts#L31-L36) +backend plugins. + +Once the reader instance is available inside the plugin, one of its methods can +directly be used with a URL. Some example usages - + +- [`read`](https://github.com/backstage/backstage/blob/d5c83bb889b8142e343ebc4e4c0b90a02d1c1a3d/plugins/catalog-backend/src/ingestion/processors/codeowners/read.ts#L24-L33) - + Catalog using the `read` method to read the CODEOWNERS file in a repository. +- [`readTree`](https://github.com/backstage/backstage/blob/d5c83bb889b8142e343ebc4e4c0b90a02d1c1a3d/packages/techdocs-common/src/helpers.ts#L198-L220) - + TechDocs using the `readTree` method to download markdown files in order to + generate the documentation site. +- [`readTree`](https://github.com/backstage/backstage/blob/d5c83bb889b8142e343ebc4e4c0b90a02d1c1a3d/packages/techdocs-common/src/stages/prepare/url.ts#L33-L54) - + TechDocs using `NotModifiedError` to maintain cache and speed up and limit the + number of requests. +- [`search`](https://github.com/backstage/backstage/blob/d5c83bb889b8142e343ebc4e4c0b90a02d1c1a3d/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.ts#L88-L108) - + Catalog using the `search` method to find files for a location URL containing + a glob pattern. + +## Writing a new URL Reader + +If the available URL Readers are not sufficient for your use case and you want +to add a new URL Reader for any other provider, you are most welcome to +contribute one! + +Feel free to use the +[GitHub URL Reader](https://github.com/backstage/backstage/blob/d5c83bb889b8142e343ebc4e4c0b90a02d1c1a3d/packages/backend-common/src/reading/GithubUrlReader.ts) +as a source of inspiration. + +### 1. Add an integration + +The provider for your new URL Reader can also be called an "integration" in +Backstage. The `integrations:` section of your Backstage `app-config.yaml` +config file is supposed to be the place where a Backstage integrator defines the +host URL for the integration, authentication details and other integration +related configurations. + +The `@backstage/integration` package is where most of the integration specific +code lives, so that it is shareable across Backstage. Functions like "read the +integrations config and process it", "construct headers for authenticated +requests to the host" or "convert a plain file URL into its API URL for +downloading the file" would live in this package. + +### 2. Create the URL Reader + +Create a new class which implements the +[`UrlReader` type](https://github.com/backstage/backstage/blob/d5c83bb889b8142e343ebc4e4c0b90a02d1c1a3d/packages/backend-common/src/reading/types.ts#L21-L28) +inside `@backstage/backend-common`. Create and export a static `factory` method +which reads the integration config and returns a map of host URLs the new reader +should be used for. See the +[GitHub URL Reader](https://github.com/backstage/backstage/blob/d5c83bb889b8142e343ebc4e4c0b90a02d1c1a3d/packages/backend-common/src/reading/GithubUrlReader.ts#L50-L63) +for example. + +### 3. Implement the methods + +We want to make sure all URL Readers behave in the same way. Hence if possible, +all the methods of the `UrlReader` interface should be implemented. However it +is okay to start by implementing just one of them and create issues for the +remaining. + +#### read + +NOTE: Use `readUrl` instead of `read`. + +`read` method expects a user-friendly URL, something which can be copied from +the browser naturally when a person is browsing the provider in their browser. + +- ✅ Valid URL : + `https://github.com/backstage/backstage/blob/master/ADOPTERS.md` +- ❌ Not a valid URL : + `https://raw.githubusercontent.com/backstage/backstage/master/ADOPTERS.md` +- ❌ Not a valid URL : `https://github.com/backstage/backstage/ADOPTERS.md` + +Upon receiving the URL, `read` converts the user-friendly URL into an API URL +which can be used to request the provider's API. + +`read` then makes an authenticated request to the provider API and returns the +file's content. + +#### readUrl + +`readUrl` is a new interface that allows complex response objects and is +intended to replace the `read` method. This new method is currently optional to +implement which allows for a soft migration to `readUrl` instead of `read` in +the future. + +#### readTree + +`readTree` method also expects user-friendly URLs similar to `read` but the URL +should point to a tree (could be the root of a repository or even a +sub-directory). + +- ✅ Valid URL : `https://github.com/backstage/backstage` +- ✅ Valid URL : `https://github.com/backstage/backstage/blob/master` +- ✅ Valid URL : `https://github.com/backstage/backstage/blob/master/docs` + +Using the provider's API documentation, find out an API endpoint which can be +used to download either a zip or a tarball. You can download the entire tree +(e.g. a repository) and filter out in case the user is expecting only a +sub-tree. But some APIs are smart enough to accept a path and return only a +sub-tree in the downloaded archive. + +#### search + +`search` method expects a glob pattern of a URL and returns a list of files +matching the query. + +- ✅ Valid URL : + `https://github.com/backstage/backstage/blob/master/**/catalog-info.yaml` +- ✅ Valid URL : `https://github.com/backstage/backstage/blob/master/**/*.md` +- ✅ Valid URL : + `https://github.com/backstage/backstage/blob/master/*/package.json` +- ✅ Valid URL : `https://github.com/backstage/backstage/blob/master/READM` + +The core logic of `readTree` can be used here to extract all the files inside +the tree and return the files matching the pattern in the `url`. + +### 4. Add to available URL Readers + +There are two ways to make your new URL Reader available for use. + +You can choose to make it open source, by updating the +[`default` factory](https://github.com/backstage/backstage/blob/d5c83bb889b8142e343ebc4e4c0b90a02d1c1a3d/packages/backend-common/src/reading/UrlReaders.ts#L62-L81) +method of URL Readers. + +But for something internal which you don't want to make open source, you can +update your `packages/backend/src/index.ts` file and update how the `reader` +instance is created. + +```ts +// File: packages/backend/src/index.ts +const reader = UrlReaders.default({ + logger: root, + config, + // This is where your internal URL Readers would go. + factories: [myCustomReader.factory], +}); +``` + +### 5. Caching + +All of the methods above support an ETag based caching. If the method is called +without an `etag`, the response contains an ETag of the resource (should ideally +forward the ETag returned by the provider). If the method is called with an +`etag`, it first compares the ETag and returns a `NotModifiedError` in case the +resource has not been modified. This approach is very similar to the actual +[ETag](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/ETag) and +[If-None-Match](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/If-None-Match) +HTTP headers. + +### 6. Debugging + +When debugging one of the URL Readers, you can straightforward use the +[`reader` instance created](https://github.com/backstage/backstage/blob/ebbe91dbe79038a61d35cf6ed2d96e0e0d5a15f3/packages/backend/src/index.ts#L57) +when the backend starts and call one of the methods with your debugging URL. + +```ts +// File: packages/backend/src/index.ts + +async function main() { + // ... + const createEnv = makeCreateEnv(config); + + const testReader = createEnv('test-url-reader').reader; + const response = await testReader.readUrl( + 'https://github.com/backstage/backstage/blob/master/catalog-info.yaml', + ); + console.log((await response.buffer()).toString()); + // ... +} +``` + +This will be run every time you restart the backend. Note that after any change +in the URL Reader code, you need to kill the backend and restart, since the +`reader` instance is memoized and does not update on hot module reloading. Also, +there are a lot of unit tests written for the URL Readers, which you can make +use of. diff --git a/microsite/blog/assets/21-06-24/backstage-search-platform.png b/microsite/blog/assets/21-06-24/backstage-search-platform.png index bb7e828ea9..b83b3aa0f4 100644 Binary files a/microsite/blog/assets/21-06-24/backstage-search-platform.png and b/microsite/blog/assets/21-06-24/backstage-search-platform.png differ diff --git a/microsite/blog/assets/21-06-24/jobs-to-be-done.png b/microsite/blog/assets/21-06-24/jobs-to-be-done.png index da86befcc2..74585d0945 100644 Binary files a/microsite/blog/assets/21-06-24/jobs-to-be-done.png and b/microsite/blog/assets/21-06-24/jobs-to-be-done.png differ diff --git a/microsite/blog/assets/21-06-24/search-results.png b/microsite/blog/assets/21-06-24/search-results.png index 041f86488c..5524d89d7e 100644 Binary files a/microsite/blog/assets/21-06-24/search-results.png and b/microsite/blog/assets/21-06-24/search-results.png differ diff --git a/microsite/data/plugins/harbor.yaml b/microsite/data/plugins/harbor.yaml new file mode 100644 index 0000000000..a851f99545 --- /dev/null +++ b/microsite/data/plugins/harbor.yaml @@ -0,0 +1,13 @@ +--- +title: Harbor +author: BESTSELLER +authorUrl: bestsellerit.com +category: Discovery +description: This plugin will show you information about docker images within harbor. +documentation: https://github.com/BESTSELLER/backstage-plugin-harbor/blob/master/README.md +iconUrl: https://bestsellerit.com/img/terraform-harbor/goharbor.jpeg +npmPackageName: '@bestsellerit/backstage-plugin-harbor' +tags: + - goharbor + - harbor + - docker diff --git a/microsite/sidebars.json b/microsite/sidebars.json index 0231d8f692..5999524073 100644 --- a/microsite/sidebars.json +++ b/microsite/sidebars.json @@ -166,7 +166,8 @@ "plugins/proxying", "plugins/backend-plugin", "plugins/call-existing-api", - "plugins/github-apps" + "plugins/github-apps", + "plugins/url-reader" ] }, { diff --git a/microsite/static/css/custom.css b/microsite/static/css/custom.css index 6404b25251..2c8b46ff00 100644 --- a/microsite/static/css/custom.css +++ b/microsite/static/css/custom.css @@ -137,6 +137,20 @@ td { color: $navigatorItemTextColor; } +/* search */ +[aria-expanded='true'] ~ .aa-without-1 { + display: block !important; +} + +[aria-expanded='true'] ~ .aa-without-1 .aa-dataset-1:after { + content: 'No results'; +} + +.navSearchWrapper { + /* Prevents the search box from detaching from its icon when pushed down */ + justify-content: flex-start; +} + /* header */ .fixedHeaderContainer { position: fixed; @@ -146,6 +160,12 @@ td { box-shadow: 3px 3px 8px rgba(38, 38, 38, 0.25); color: #fff; } + +.wrapper.headerWrapper { + /* Keep the header stretched across the screen on the mid breakpoint, since we need a bit more space */ + max-width: 1400px; +} + @media only screen and (min-width: 1024px) { .fixedHeaderContainer { height: 66px; diff --git a/mkdocs.yml b/mkdocs.yml index c3d77e179f..4d984e63e3 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -112,6 +112,7 @@ nav: - Backend plugin: 'plugins/backend-plugin.md' - Call existing API: 'plugins/call-existing-api.md' - GitHub Apps for Backend Authentication: 'plugins/github-apps.md' + - URL Reader: 'plugins/url-reader.md' - Testing: - Testing with Jest: 'plugins/testing.md' - Publishing: diff --git a/package.json b/package.json index c0887d7f05..09dc55617b 100644 --- a/package.json +++ b/package.json @@ -61,7 +61,7 @@ "command-exists": "^1.2.9", "concurrently": "^6.0.0", "eslint-plugin-notice": "^0.9.10", - "fs-extra": "^9.0.0", + "fs-extra": "^10.0.0", "husky": "^6.0.0", "lerna": "^4.0.0", "lint-staged": "^10.1.0", diff --git a/packages/app/src/components/Root/Root.tsx b/packages/app/src/components/Root/Root.tsx index 09ee571cf6..48141e4f53 100644 --- a/packages/app/src/components/Root/Root.tsx +++ b/packages/app/src/components/Root/Root.tsx @@ -39,6 +39,7 @@ import { SidebarItem, SidebarDivider, SidebarSpace, + SidebarScrollWrapper, } from '@backstage/core-components'; const useSidebarLogoStyles = makeStyles({ @@ -88,10 +89,12 @@ export const Root = ({ children }: PropsWithChildren<{}>) => ( {/* End global nav */} - - - - + + + + + + diff --git a/packages/backend-common/api-report.md b/packages/backend-common/api-report.md index 6b9e7bbbe0..a21aaef4a6 100644 --- a/packages/backend-common/api-report.md +++ b/packages/backend-common/api-report.md @@ -42,6 +42,8 @@ export class AzureUrlReader implements UrlReader { // (undocumented) readTree(url: string, options?: ReadTreeOptions): Promise; // (undocumented) + readUrl(url: string, _options?: ReadUrlOptions): Promise; + // (undocumented) search(url: string, options?: SearchOptions): Promise; // (undocumented) toString(): string; @@ -59,6 +61,8 @@ export class BitbucketUrlReader implements UrlReader { // (undocumented) readTree(url: string, options?: ReadTreeOptions): Promise; // (undocumented) + readUrl(url: string, _options?: ReadUrlOptions): Promise; + // (undocumented) search(url: string, options?: SearchOptions): Promise; // (undocumented) toString(): string; @@ -184,8 +188,9 @@ export class Git { logger?: Logger | undefined; }) => Git; // (undocumented) - init({ dir }: { + init({ dir, defaultBranch, }: { dir: string; + defaultBranch?: string; }): Promise; // (undocumented) merge({ dir, theirs, ours, author, committer, }: { @@ -231,6 +236,8 @@ export class GithubUrlReader implements UrlReader { // (undocumented) readTree(url: string, options?: ReadTreeOptions): Promise; // (undocumented) + readUrl(url: string, options?: ReadUrlOptions): Promise; + // (undocumented) search(url: string, options?: SearchOptions): Promise; // (undocumented) toString(): string; @@ -248,6 +255,8 @@ export class GitlabUrlReader implements UrlReader { // (undocumented) readTree(url: string, options?: ReadTreeOptions): Promise; // (undocumented) + readUrl(url: string, options?: ReadUrlOptions): Promise; + // (undocumented) search(url: string, options?: SearchOptions): Promise; // (undocumented) toString(): string; @@ -277,7 +286,7 @@ export type PluginEndpointDiscovery = { getExternalBaseUrl(pluginId: string): Promise; }; -// @public (undocumented) +// @public export type ReadTreeResponse = { files(): Promise; archive(): Promise; @@ -366,6 +375,7 @@ export interface StatusCheckHandlerOptions { // @public export type UrlReader = { read(url: string): Promise; + readUrl?(url: string, options?: ReadUrlOptions): Promise; readTree(url: string, options?: ReadTreeOptions): Promise; search(url: string, options?: SearchOptions): Promise; }; diff --git a/packages/backend-common/package.json b/packages/backend-common/package.json index d395b8d557..fdbea23bd8 100644 --- a/packages/backend-common/package.json +++ b/packages/backend-common/package.json @@ -47,7 +47,7 @@ "dockerode": "^3.2.1", "express": "^4.17.1", "express-promise-router": "^4.1.0", - "fs-extra": "^9.0.1", + "fs-extra": "^10.0.0", "git-url-parse": "^11.4.4", "helmet": "^4.0.0", "isomorphic-git": "^1.8.0", diff --git a/packages/backend-common/src/reading/AzureUrlReader.ts b/packages/backend-common/src/reading/AzureUrlReader.ts index fe541538ea..bc22854d4e 100644 --- a/packages/backend-common/src/reading/AzureUrlReader.ts +++ b/packages/backend-common/src/reading/AzureUrlReader.ts @@ -36,6 +36,8 @@ import { SearchOptions, SearchResponse, UrlReader, + ReadUrlOptions, + ReadUrlResponse, } from './types'; export class AzureUrlReader implements UrlReader { @@ -78,6 +80,15 @@ export class AzureUrlReader implements UrlReader { throw new Error(message); } + async readUrl( + url: string, + _options?: ReadUrlOptions, + ): Promise { + // TODO etag is not implemented yet. + const buffer = await this.read(url); + return { buffer: async () => buffer }; + } + async readTree( url: string, options?: ReadTreeOptions, diff --git a/packages/backend-common/src/reading/BitbucketUrlReader.test.ts b/packages/backend-common/src/reading/BitbucketUrlReader.test.ts index cc540cb83c..b032322627 100644 --- a/packages/backend-common/src/reading/BitbucketUrlReader.test.ts +++ b/packages/backend-common/src/reading/BitbucketUrlReader.test.ts @@ -74,7 +74,24 @@ describe('BitbucketUrlReader', () => { const worker = setupServer(); msw.setupDefaultHandlers(worker); - describe('implementation', () => { + describe('readUrl', () => { + worker.use( + rest.get( + 'https://api.bitbucket.org/2.0/repositories/backstage-verification/test-template/src/master/template.yaml', + (_, res, ctx) => res(ctx.status(200), ctx.body('foo')), + ), + ); + + it('should be able to readUrl', async () => { + const result = await bitbucketProcessor.readUrl( + 'https://bitbucket.org/backstage-verification/test-template/src/master/template.yaml', + ); + const buffer = await result.buffer(); + expect(buffer.toString()).toBe('foo'); + }); + }); + + describe('read', () => { it('rejects unknown targets', async () => { await expect( bitbucketProcessor.read('https://not.bitbucket.com/apa'), @@ -119,14 +136,14 @@ describe('BitbucketUrlReader', () => { ), ), rest.get( - 'https://bitbucket.org/backstage/mock/get/master.tgz', + 'https://bitbucket.org/backstage/mock/get/master.tar.gz', (_, res, ctx) => res( ctx.status(200), ctx.set('Content-Type', 'application/zip'), ctx.set( 'content-disposition', - 'attachment; filename=backstage-mock-12ab34cd56ef.tgz', + 'attachment; filename=backstage-mock-12ab34cd56ef.tar.gz', ), ctx.body(repoBuffer), ), @@ -304,14 +321,14 @@ describe('BitbucketUrlReader', () => { ), ), rest.get( - 'https://bitbucket.org/backstage/mock/get/master.tgz', + 'https://bitbucket.org/backstage/mock/get/master.tar.gz', (_, res, ctx) => res( ctx.status(200), ctx.set('Content-Type', 'application/zip'), ctx.set( 'content-disposition', - 'attachment; filename=backstage-mock-12ab34cd56ef.tgz', + 'attachment; filename=backstage-mock-12ab34cd56ef.tar.gz', ), ctx.body(repoBuffer), ), diff --git a/packages/backend-common/src/reading/BitbucketUrlReader.ts b/packages/backend-common/src/reading/BitbucketUrlReader.ts index c4dc6e135f..2849bc3cd9 100644 --- a/packages/backend-common/src/reading/BitbucketUrlReader.ts +++ b/packages/backend-common/src/reading/BitbucketUrlReader.ts @@ -36,6 +36,8 @@ import { SearchOptions, SearchResponse, UrlReader, + ReadUrlResponse, + ReadUrlOptions, } from './types'; /** @@ -99,6 +101,15 @@ export class BitbucketUrlReader implements UrlReader { throw new Error(message); } + async readUrl( + url: string, + _options?: ReadUrlOptions, + ): Promise { + // TODO etag is not implemented yet. + const buffer = await this.read(url); + return { buffer: async () => buffer }; + } + async readTree( url: string, options?: ReadTreeOptions, diff --git a/packages/backend-common/src/reading/FetchUrlReader.ts b/packages/backend-common/src/reading/FetchUrlReader.ts index 4c03ea904d..30468158aa 100644 --- a/packages/backend-common/src/reading/FetchUrlReader.ts +++ b/packages/backend-common/src/reading/FetchUrlReader.ts @@ -19,6 +19,8 @@ import { NotFoundError } from '@backstage/errors'; import { ReaderFactory, ReadTreeResponse, + ReadUrlOptions, + ReadUrlResponse, SearchResponse, UrlReader, } from './types'; @@ -73,6 +75,15 @@ export class FetchUrlReader implements UrlReader { throw new Error(message); } + async readUrl( + url: string, + _options?: ReadUrlOptions, + ): Promise { + // TODO etag is not implemented yet. + const buffer = await this.read(url); + return { buffer: async () => buffer }; + } + async readTree(): Promise { throw new Error('FetchUrlReader does not implement readTree'); } diff --git a/packages/backend-common/src/reading/GithubUrlReader.test.ts b/packages/backend-common/src/reading/GithubUrlReader.test.ts index 0b3288e59c..c8002cd3ce 100644 --- a/packages/backend-common/src/reading/GithubUrlReader.test.ts +++ b/packages/backend-common/src/reading/GithubUrlReader.test.ts @@ -141,6 +141,113 @@ describe('GithubUrlReader', () => { }); }); + /* + * readUrl + */ + describe('readUrl', () => { + it('should use the headers from the credentials provider to the fetch request when doing readUrl', async () => { + expect.assertions(2); + + const mockHeaders = { + Authorization: 'bearer blah', + otherheader: 'something', + }; + + (mockCredentialsProvider.getCredentials as jest.Mock).mockResolvedValue({ + headers: mockHeaders, + }); + + worker.use( + rest.get( + 'https://ghe.github.com/api/v3/repos/backstage/mock/tree/contents/', + (req, res, ctx) => { + expect(req.headers.get('authorization')).toBe( + mockHeaders.Authorization, + ); + expect(req.headers.get('otherheader')).toBe( + mockHeaders.otherheader, + ); + return res( + ctx.status(200), + ctx.set('Content-Type', 'application/x-gzip'), + ctx.body('foo'), + ); + }, + ), + ); + + await gheProcessor.readUrl( + 'https://github.com/backstage/mock/tree/blob/main', + ); + }); + + it('should throw NotModified if GitHub responds with 304', async () => { + expect.assertions(4); + + const mockHeaders = { + Authorization: 'bearer blah', + otherheader: 'something', + }; + + (mockCredentialsProvider.getCredentials as jest.Mock).mockResolvedValue({ + headers: mockHeaders, + }); + + worker.use( + rest.get( + 'https://ghe.github.com/api/v3/repos/backstage/mock/tree/contents/', + (req, res, ctx) => { + expect(req.headers.get('authorization')).toBe( + mockHeaders.Authorization, + ); + expect(req.headers.get('otherheader')).toBe( + mockHeaders.otherheader, + ); + expect(req.headers.get('if-none-match')).toBe('foo'); + return res( + ctx.status(304), + ctx.set('Content-Type', 'application/x-gzip'), + ctx.body('foo'), + ); + }, + ), + ); + + await expect( + gheProcessor.readUrl( + 'https://github.com/backstage/mock/tree/blob/main', + { etag: 'foo' }, + ), + ).rejects.toThrow(NotModifiedError); + }); + + it('should return etag from the response', async () => { + (mockCredentialsProvider.getCredentials as jest.Mock).mockResolvedValue({ + headers: { + Authorization: 'bearer blah', + }, + }); + + worker.use( + rest.get( + 'https://ghe.github.com/api/v3/repos/backstage/mock/tree/contents/', + (_req, res, ctx) => { + return res( + ctx.status(200), + ctx.set('Etag', 'foo'), + ctx.body('bar'), + ); + }, + ), + ); + + const response = await gheProcessor.readUrl( + 'https://github.com/backstage/mock/tree/blob/main', + ); + expect(response.etag).toBe('foo'); + }); + }); + /* * readTree */ diff --git a/packages/backend-common/src/reading/GithubUrlReader.ts b/packages/backend-common/src/reading/GithubUrlReader.ts index 70bc9601df..bccddc6838 100644 --- a/packages/backend-common/src/reading/GithubUrlReader.ts +++ b/packages/backend-common/src/reading/GithubUrlReader.ts @@ -35,6 +35,8 @@ import { SearchResponse, SearchResponseFile, UrlReader, + ReadUrlOptions, + ReadUrlResponse, } from './types'; export type GhRepoResponse = RestEndpointMethodTypes['repos']['get']['response']['data']; @@ -77,6 +79,14 @@ export class GithubUrlReader implements UrlReader { } async read(url: string): Promise { + const response = await this.readUrl(url); + return response.buffer(); + } + + async readUrl( + url: string, + options?: ReadUrlOptions, + ): Promise { const ghUrl = getGitHubFileFetchUrl(url, this.integration.config); const { headers } = await this.deps.credentialsProvider.getCredentials({ url, @@ -86,6 +96,7 @@ export class GithubUrlReader implements UrlReader { response = await fetch(ghUrl.toString(), { headers: { ...headers, + ...(options?.etag && { 'If-None-Match': options.etag }), Accept: 'application/vnd.github.v3.raw', }, }); @@ -93,8 +104,15 @@ export class GithubUrlReader implements UrlReader { throw new Error(`Unable to read ${url}, ${e}`); } + if (response.status === 304) { + throw new NotModifiedError(); + } + if (response.ok) { - return Buffer.from(await response.text()); + return { + buffer: async () => Buffer.from(await response.text()), + etag: response.headers.get('ETag') ?? undefined, + }; } const message = `${url} could not be read as ${ghUrl}, ${response.status} ${response.statusText}`; diff --git a/packages/backend-common/src/reading/GitlabUrlReader.test.ts b/packages/backend-common/src/reading/GitlabUrlReader.test.ts index 9c2e428239..945444d8ed 100644 --- a/packages/backend-common/src/reading/GitlabUrlReader.test.ts +++ b/packages/backend-common/src/reading/GitlabUrlReader.test.ts @@ -79,7 +79,7 @@ describe('GitlabUrlReader', () => { const worker = setupServer(); msw.setupDefaultHandlers(worker); - describe('implementation', () => { + describe('read', () => { beforeEach(() => { worker.use( rest.get('*/api/v4/projects/:name', (_, res, ctx) => @@ -180,6 +180,53 @@ describe('GitlabUrlReader', () => { }); }); + describe('readUrl', () => { + const [{ reader }] = GitlabUrlReader.factory({ + config: new ConfigReader({}), + logger, + treeResponseFactory, + }); + + it('should throw NotModified on HTTP 304', async () => { + worker.use( + rest.get('*/api/v4/projects/:name', (_, res, ctx) => + res(ctx.status(200), ctx.json({ id: 12345 })), + ), + rest.get('*', (req, res, ctx) => { + expect(req.headers.get('If-None-Match')).toBe('999'); + return res(ctx.status(304)); + }), + ); + + await expect( + reader.readUrl!( + 'https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/my/path/to/file.yaml', + { + etag: '999', + }, + ), + ).rejects.toThrow(NotModifiedError); + }); + + it('should return etag in response', async () => { + worker.use( + rest.get('*/api/v4/projects/:name', (_, res, ctx) => + res(ctx.status(200), ctx.json({ id: 12345 })), + ), + rest.get('*', (_req, res, ctx) => { + return res(ctx.status(200), ctx.set('ETag', '999'), ctx.body('foo')); + }), + ); + + const result = await reader.readUrl!( + 'https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/my/path/to/file.yaml', + ); + expect(result.etag).toBe('999'); + const content = await result.buffer(); + expect(content.toString()).toBe('foo'); + }); + }); + describe('readTree', () => { const archiveBuffer = fs.readFileSync( path.resolve('src', 'reading', '__fixtures__', 'gitlab-archive.tar.gz'), diff --git a/packages/backend-common/src/reading/GitlabUrlReader.ts b/packages/backend-common/src/reading/GitlabUrlReader.ts index aa215aee47..6c272471e2 100644 --- a/packages/backend-common/src/reading/GitlabUrlReader.ts +++ b/packages/backend-common/src/reading/GitlabUrlReader.ts @@ -34,6 +34,8 @@ import { SearchOptions, SearchResponse, UrlReader, + ReadUrlResponse, + ReadUrlOptions, } from './types'; export class GitlabUrlReader implements UrlReader { @@ -54,20 +56,37 @@ export class GitlabUrlReader implements UrlReader { ) {} async read(url: string): Promise { + const response = await this.readUrl(url); + return response.buffer(); + } + + async readUrl( + url: string, + options?: ReadUrlOptions, + ): Promise { const builtUrl = await getGitLabFileFetchUrl(url, this.integration.config); let response: Response; try { - response = await fetch( - builtUrl, - getGitLabRequestOptions(this.integration.config), - ); + response = await fetch(builtUrl, { + headers: { + ...getGitLabRequestOptions(this.integration.config).headers, + ...(options?.etag && { 'If-None-Match': options.etag }), + }, + }); } catch (e) { throw new Error(`Unable to read ${url}, ${e}`); } + if (response.status === 304) { + throw new NotModifiedError(); + } + if (response.ok) { - return Buffer.from(await response.text()); + return { + buffer: async () => Buffer.from(await response.text()), + etag: response.headers.get('ETag') ?? undefined, + }; } const message = `${url} could not be read as ${builtUrl}, ${response.status} ${response.statusText}`; diff --git a/packages/backend-common/src/reading/GoogleGcsUrlReader.ts b/packages/backend-common/src/reading/GoogleGcsUrlReader.ts index e06612cd15..9f3fbad302 100644 --- a/packages/backend-common/src/reading/GoogleGcsUrlReader.ts +++ b/packages/backend-common/src/reading/GoogleGcsUrlReader.ts @@ -18,6 +18,8 @@ import { Storage } from '@google-cloud/storage'; import { ReaderFactory, ReadTreeResponse, + ReadUrlOptions, + ReadUrlResponse, SearchResponse, UrlReader, } from './types'; @@ -90,6 +92,15 @@ export class GoogleGcsUrlReader implements UrlReader { } } + async readUrl( + url: string, + _options?: ReadUrlOptions, + ): Promise { + // TODO etag is not implemented yet. + const buffer = await this.read(url); + return { buffer: async () => buffer }; + } + async readTree(): Promise { throw new Error('GcsUrlReader does not implement readTree'); } diff --git a/packages/backend-common/src/reading/UrlReaderPredicateMux.ts b/packages/backend-common/src/reading/UrlReaderPredicateMux.ts index c9afecf907..4a166cb517 100644 --- a/packages/backend-common/src/reading/UrlReaderPredicateMux.ts +++ b/packages/backend-common/src/reading/UrlReaderPredicateMux.ts @@ -15,27 +15,35 @@ */ import { NotAllowedError } from '@backstage/errors'; +import { Logger } from 'winston'; import { ReadTreeOptions, ReadTreeResponse, + ReadUrlOptions, + ReadUrlResponse, SearchOptions, SearchResponse, UrlReader, UrlReaderPredicateTuple, } from './types'; +const MIN_WARNING_INTERVAL_MS = 1000 * 60 * 15; + /** * A UrlReader implementation that selects from a set of UrlReaders * based on a predicate tied to each reader. */ export class UrlReaderPredicateMux implements UrlReader { private readonly readers: UrlReaderPredicateTuple[] = []; + private readonly readerWarnings: Map = new Map(); + + constructor(private readonly logger: Logger) {} register(tuple: UrlReaderPredicateTuple): void { this.readers.push(tuple); } - read(url: string): Promise { + async read(url: string): Promise { const parsed = new URL(url); for (const { predicate, reader } of this.readers) { @@ -47,6 +55,37 @@ export class UrlReaderPredicateMux implements UrlReader { throw new NotAllowedError(`Reading from '${url}' is not allowed`); } + async readUrl( + url: string, + options?: ReadUrlOptions, + ): Promise { + const parsed = new URL(url); + + for (const { predicate, reader } of this.readers) { + if (predicate(parsed)) { + if (reader.readUrl) { + return reader.readUrl(url, options); + } + const now = Date.now(); + const lastWarned = this.readerWarnings.get(reader) ?? 0; + if (now > lastWarned + MIN_WARNING_INTERVAL_MS) { + this.readerWarnings.set(reader, now); + this.logger.warn( + `No implementation of readUrl found for ${reader}, this method will be required in the ` + + `future and will replace the 'read' method. See the changelog for more details here: ` + + 'https://github.com/backstage/backstage/blob/master/packages/backend-common/CHANGELOG.md#085', + ); + } + const buffer = await reader.read(url); + return { + buffer: async () => buffer, + }; + } + } + + throw new NotAllowedError(`Reading from '${url}' is not allowed`); + } + async readTree( url: string, options?: ReadTreeOptions, diff --git a/packages/backend-common/src/reading/UrlReaders.ts b/packages/backend-common/src/reading/UrlReaders.ts index bc77877384..2b3a2f166c 100644 --- a/packages/backend-common/src/reading/UrlReaders.ts +++ b/packages/backend-common/src/reading/UrlReaders.ts @@ -43,7 +43,7 @@ export class UrlReaders { * Creates a UrlReader without any known types. */ static create({ logger, config, factories }: CreateOptions): UrlReader { - const mux = new UrlReaderPredicateMux(); + const mux = new UrlReaderPredicateMux(logger); const treeResponseFactory = DefaultReadTreeResponseFactory.create({ config, }); diff --git a/packages/backend-common/src/reading/types.ts b/packages/backend-common/src/reading/types.ts index f7fad4dd7a..8efc833ead 100644 --- a/packages/backend-common/src/reading/types.ts +++ b/packages/backend-common/src/reading/types.ts @@ -22,8 +22,20 @@ import { Config } from '@backstage/config'; * A generic interface for fetching plain data from URLs. */ export type UrlReader = { + /* Used to read a single file and return its content. */ read(url: string): Promise; + + /** + * A replacement for the read method that supports options and complex responses. + * + * Use this whenever it is available, as the read method will be deprecated and + * eventually removed in the future. + */ + readUrl?(url: string, options?: ReadUrlOptions): Promise; + + /* Used to read a file tree and download as a directory. */ readTree(url: string, options?: ReadTreeOptions): Promise; + /* Used to search a file in a tree using a glob pattern. */ search(url: string, options?: SearchOptions): Promise; }; @@ -42,6 +54,40 @@ export type ReaderFactory = (options: { treeResponseFactory: ReadTreeResponseFactory; }) => UrlReaderPredicateTuple[]; +/** + * An options object for readUrl operations. + */ +export type ReadUrlOptions = { + /** + * An etag can be provided to check whether readUrl's response has changed from a previous execution. + * + * In the readUrl() response, an etag is returned along with the data. The etag is a unique identifer + * of the data, usually the commit SHA or etag from the target. + * + * When an etag is given in ReadUrlOptions, readUrl will first compare the etag against the etag + * on the target. If they match, readUrl will throw a NotModifiedError indicating that the readUrl + * response will not differ from the previous response which included this particular etag. If they + * do not match, readUrl will return the rest of ReadUrlResponse along with a new etag. + */ + etag?: string; +}; + +/** + * A response object for readUrl operations. + */ +export type ReadUrlResponse = { + /** + * Returns the data that was read from the remote URL. + */ + buffer(): Promise; + + /** + * Etag returned by content provider. + * Can be used to compare and cache responses when doing subsequent calls. + */ + etag?: string; +}; + /** * An options object for readTree operations. */ @@ -66,14 +112,17 @@ export type ReadTreeOptions = { * In the readTree() response, an etag is returned along with the tree blob. The etag is a unique identifer * of the tree blob, usually the commit SHA or etag from the target. * - * When a etag is given in ReadTreeOptions, readTree will first compare the etag against the etag + * When an etag is given in ReadTreeOptions, readTree will first compare the etag against the etag * on the target branch. If they match, readTree will throw a NotModifiedError indicating that the readTree - * response will not differ from the previous response which included this particular etag. If they mismatch, - * readTree will return the rest of ReadTreeResponse along with a new etag. + * response will not differ from the previous response which included this particular etag. If they + * do not match, readTree will return the rest of ReadTreeResponse along with a new etag. */ etag?: string; }; +/** + * A response object for readTree operations. + */ export type ReadTreeResponse = { /** * files() returns an array of all the files inside the tree and corresponding functions to read their content. @@ -87,7 +136,8 @@ export type ReadTreeResponse = { dir(options?: ReadTreeResponseDirOptions): Promise; /** - * A unique identifer of the tree blob, usually the commit SHA or etag from the target. + * Etag returned by content provider. + * Can be used to compare and cache responses when doing subsequent calls. */ etag: string; }; diff --git a/packages/backend-common/src/scm/git.test.ts b/packages/backend-common/src/scm/git.test.ts index af9dfeef1b..0b70edc45d 100644 --- a/packages/backend-common/src/scm/git.test.ts +++ b/packages/backend-common/src/scm/git.test.ts @@ -201,14 +201,16 @@ describe('Git', () => { describe('init', () => { it('should call isomorphic-git with the correct arguments', async () => { const dir = '/some/mock/dir'; + const defaultBranch = 'master'; const git = Git.fromAuth({}); - await git.init({ dir }); + await git.init({ dir, defaultBranch }); expect(isomorphic.init).toHaveBeenCalledWith({ fs, dir, + defaultBranch, }); }); }); diff --git a/packages/backend-common/src/scm/git.ts b/packages/backend-common/src/scm/git.ts index 3afa551d5f..e786c279a1 100644 --- a/packages/backend-common/src/scm/git.ts +++ b/packages/backend-common/src/scm/git.ts @@ -150,12 +150,19 @@ export class Git { }); } - async init({ dir }: { dir: string }): Promise { + async init({ + dir, + defaultBranch = 'master', + }: { + dir: string; + defaultBranch?: string; + }): Promise { this.config.logger?.info(`Init git repository {dir=${dir}}`); return git.init({ fs, dir, + defaultBranch, }); } diff --git a/packages/catalog-model/src/validation/KubernetesValidatorFunctions.test.ts b/packages/catalog-model/src/validation/KubernetesValidatorFunctions.test.ts index f9b2b6b957..c194774906 100644 --- a/packages/catalog-model/src/validation/KubernetesValidatorFunctions.test.ts +++ b/packages/catalog-model/src/validation/KubernetesValidatorFunctions.test.ts @@ -76,9 +76,10 @@ describe('KubernetesValidatorFunctions', () => { ['a-b', true], ['-a-b', false], ['a-b-', false], - ['a--b', false], + ['a--b', true], ['a_b', true], ['a.b', true], + ['a..b', true], ])(`isValidObjectName %p ? %p`, (value, matches) => { expect(KubernetesValidatorFunctions.isValidObjectName(value)).toBe(matches); }); @@ -114,9 +115,10 @@ describe('KubernetesValidatorFunctions', () => { ['a-b', true], ['-a-b', false], ['a-b-', false], - ['a--b', false], + ['a--b', true], ['a_b', true], ['a.b', true], + ['a..b', true], ['a/a', true], ['a-b.c/a', true], ['a--b.c/a', false], @@ -150,9 +152,10 @@ describe('KubernetesValidatorFunctions', () => { ['a-b', true], ['-a-b', false], ['a-b-', false], - ['a--b', false], + ['a--b', true], ['a_b', true], ['a.b', true], + ['a..b', true], ])(`isValidLabelValue %p ? %p`, (value, matches) => { expect(KubernetesValidatorFunctions.isValidLabelValue(value)).toBe(matches); }); @@ -169,9 +172,10 @@ describe('KubernetesValidatorFunctions', () => { ['a-b', true], ['-a-b', false], ['a-b-', false], - ['a--b', false], + ['a--b', true], ['a_b', true], ['a.b', true], + ['a..b', true], ['a/a', true], ['a-b.c/a', true], ['a--b.c/a', false], diff --git a/packages/catalog-model/src/validation/KubernetesValidatorFunctions.ts b/packages/catalog-model/src/validation/KubernetesValidatorFunctions.ts index 049678e3a9..86d27e7b13 100644 --- a/packages/catalog-model/src/validation/KubernetesValidatorFunctions.ts +++ b/packages/catalog-model/src/validation/KubernetesValidatorFunctions.ts @@ -48,7 +48,7 @@ export class KubernetesValidatorFunctions { typeof value === 'string' && value.length >= 1 && value.length <= 63 && - /^[a-z0-9A-Z]+([-_.][a-z0-9A-Z]+)*$/.test(value) + /^([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$/.test(value) ); } diff --git a/packages/cli/package.json b/packages/cli/package.json index 200531a1ec..c9bde05bd7 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -77,7 +77,7 @@ "express": "^4.17.1", "file-loader": "^6.2.0", "fork-ts-checker-webpack-plugin": "^6.2.9", - "fs-extra": "^9.0.0", + "fs-extra": "^10.0.0", "handlebars": "^4.7.3", "html-webpack-plugin": "^4.3.0", "inquirer": "^7.0.4", diff --git a/packages/config-loader/package.json b/packages/config-loader/package.json index 4a0ede9873..d5a4f600fd 100644 --- a/packages/config-loader/package.json +++ b/packages/config-loader/package.json @@ -34,7 +34,7 @@ "@backstage/config": "^0.1.5", "@types/json-schema": "^7.0.6", "ajv": "^7.0.3", - "fs-extra": "^9.0.0", + "fs-extra": "^10.0.0", "json-schema": "^0.3.0", "json-schema-merge-allof": "^0.8.1", "typescript-json-schema": "^0.50.1", diff --git a/packages/core-components/src/components/Link/Link.tsx b/packages/core-components/src/components/Link/Link.tsx index dd169390a5..7853c4f52f 100644 --- a/packages/core-components/src/components/Link/Link.tsx +++ b/packages/core-components/src/components/Link/Link.tsx @@ -37,9 +37,16 @@ export type LinkProps = MaterialLinkProps & */ export const Link = React.forwardRef((props, ref) => { const to = String(props.to); - return isExternalUri(to) ? ( + const external = isExternalUri(to); + const newWindow = external && !!/^https?:/.exec(to); + return external ? ( // External links - + ) : ( // Interact with React Router for internal links diff --git a/packages/core-components/src/components/ResponseErrorPanel/ResponseErrorPanel.tsx b/packages/core-components/src/components/ResponseErrorPanel/ResponseErrorPanel.tsx index 231cd179c7..ccfb706bf9 100644 --- a/packages/core-components/src/components/ResponseErrorPanel/ResponseErrorPanel.tsx +++ b/packages/core-components/src/components/ResponseErrorPanel/ResponseErrorPanel.tsx @@ -76,7 +76,7 @@ export const ResponseErrorPanel = ({ @@ -87,10 +87,9 @@ export const ResponseErrorPanel = ({ } /> - + ); diff --git a/packages/core-components/src/layout/InfoCard/InfoCard.tsx b/packages/core-components/src/layout/InfoCard/InfoCard.tsx index 763038388b..cc15bbd9f4 100644 --- a/packages/core-components/src/layout/InfoCard/InfoCard.tsx +++ b/packages/core-components/src/layout/InfoCard/InfoCard.tsx @@ -37,7 +37,6 @@ const useStyles = makeStyles(theme => ({ }, }, header: { - display: 'inline-block', padding: theme.spacing(2, 2, 2, 2.5), }, headerTitle: { @@ -121,6 +120,7 @@ type Props = { children?: ReactNode; headerStyle?: object; headerProps?: CardHeaderProps; + action?: ReactNode; actionsClassName?: string; actions?: ReactNode; cardClassName?: string; @@ -141,6 +141,7 @@ export const InfoCard = ({ children, headerStyle, headerProps, + action, actionsClassName, actions, cardClassName, @@ -190,6 +191,7 @@ export const InfoCard = ({ }} title={title} subheader={subheader} + action={action} style={{ ...headerStyle }} titleTypographyProps={titleTypographyProps} {...headerProps} diff --git a/packages/core-components/src/layout/Page/Page.tsx b/packages/core-components/src/layout/Page/Page.tsx index 8acb54c268..8d1dfe2943 100644 --- a/packages/core-components/src/layout/Page/Page.tsx +++ b/packages/core-components/src/layout/Page/Page.tsx @@ -25,7 +25,8 @@ const useStyles = makeStyles(() => ({ "'pageHeader pageHeader pageHeader' 'pageSubheader pageSubheader pageSubheader' 'pageNav pageContent pageSidebar'", gridTemplateRows: 'auto auto 1fr', gridTemplateColumns: 'auto 1fr auto', - minHeight: '100vh', + height: '100vh', + overflowY: 'auto', }, })); diff --git a/packages/core-components/src/layout/Sidebar/Bar.tsx b/packages/core-components/src/layout/Sidebar/Bar.tsx index f6edac81b7..65da4d2d75 100644 --- a/packages/core-components/src/layout/Sidebar/Bar.tsx +++ b/packages/core-components/src/layout/Sidebar/Bar.tsx @@ -14,7 +14,7 @@ * limitations under the License. */ -import { makeStyles } from '@material-ui/core'; +import { makeStyles, useMediaQuery } from '@material-ui/core'; import clsx from 'clsx'; import React, { useRef, useState, useContext, PropsWithChildren } from 'react'; import { sidebarConfig, SidebarContext } from './config'; @@ -82,6 +82,9 @@ export const Sidebar = ({ children, }: PropsWithChildren) => { const classes = useStyles(); + const isSmallScreen = useMediaQuery(theme => + theme.breakpoints.down('md'), + ); const [state, setState] = useState(State.Closed); const hoverTimerRef = useRef(); const { isPinned } = useContext(SidebarPinStateContext); @@ -94,7 +97,7 @@ export const Sidebar = ({ clearTimeout(hoverTimerRef.current); hoverTimerRef.current = undefined; } - if (state !== State.Open) { + if (state !== State.Open && !isSmallScreen) { hoverTimerRef.current = window.setTimeout(() => { hoverTimerRef.current = undefined; setState(State.Open); @@ -122,6 +125,8 @@ export const Sidebar = ({ } }; + const isOpen = (state === State.Open && !isSmallScreen) || isPinned; + return (
{children} diff --git a/packages/core-components/src/layout/Sidebar/Items.tsx b/packages/core-components/src/layout/Sidebar/Items.tsx index 9bccffc6c4..28abbce53c 100644 --- a/packages/core-components/src/layout/Sidebar/Items.tsx +++ b/packages/core-components/src/layout/Sidebar/Items.tsx @@ -21,8 +21,10 @@ import { makeStyles, styled, TextField, + Theme, Typography, } from '@material-ui/core'; +import { CreateCSSProperties } from '@material-ui/core/styles/withStyles'; import SearchIcon from '@material-ui/icons/Search'; import clsx from 'clsx'; import React, { @@ -290,3 +292,32 @@ export const SidebarDivider = styled('hr')({ border: 'none', margin: '12px 0px', }); + +const styledScrollbar = (theme: Theme): CreateCSSProperties => ({ + overflowY: 'auto', + '&::-webkit-scrollbar': { + backgroundColor: theme.palette.background.default, + width: '5px', + borderRadius: '5px', + }, + '&::-webkit-scrollbar-thumb': { + backgroundColor: theme.palette.text.hint, + borderRadius: '5px', + }, +}); + +export const SidebarScrollWrapper = styled('div')(({ theme }) => { + const scrollbarStyles = styledScrollbar(theme); + return { + flex: '0 1 auto', + overflowX: 'hidden', + // 5px space to the right of the scrollbar + width: 'calc(100% - 5px)', + // Display at least one item in the container + // Question: Can this be a config/theme variable - if so, which? :/ + minHeight: '48px', + overflowY: 'hidden', + '@media (hover: none)': scrollbarStyles, + '&:hover': scrollbarStyles, + }; +}); diff --git a/packages/core-components/src/layout/Sidebar/index.ts b/packages/core-components/src/layout/Sidebar/index.ts index fc0e29f9a6..58e193dbaf 100644 --- a/packages/core-components/src/layout/Sidebar/index.ts +++ b/packages/core-components/src/layout/Sidebar/index.ts @@ -23,6 +23,7 @@ export { SidebarSearchField, SidebarSpace, SidebarSpacer, + SidebarScrollWrapper, } from './Items'; export { IntroCard, SidebarIntro } from './Intro'; export { diff --git a/packages/create-app/package.json b/packages/create-app/package.json index 95bb58a003..73c41a79f0 100644 --- a/packages/create-app/package.json +++ b/packages/create-app/package.json @@ -30,7 +30,7 @@ "@backstage/cli-common": "^0.1.2", "chalk": "^4.0.0", "commander": "^6.1.0", - "fs-extra": "^9.0.0", + "fs-extra": "^10.0.0", "handlebars": "^4.7.3", "inquirer": "^7.0.4", "ora": "^5.3.0", diff --git a/packages/create-app/templates/default-app/packages/app/src/components/Root/Root.tsx b/packages/create-app/templates/default-app/packages/app/src/components/Root/Root.tsx index 5e5a71a635..d8665a2f9b 100644 --- a/packages/create-app/templates/default-app/packages/app/src/components/Root/Root.tsx +++ b/packages/create-app/templates/default-app/packages/app/src/components/Root/Root.tsx @@ -34,6 +34,7 @@ import { SidebarItem, SidebarDivider, SidebarSpace, + SidebarScrollWrapper, } from '@backstage/core-components'; const useSidebarLogoStyles = makeStyles({ @@ -82,7 +83,9 @@ export const Root = ({ children }: PropsWithChildren<{}>) => ( {/* End global nav */} - + + + diff --git a/packages/docgen/package.json b/packages/docgen/package.json index 6ee4419349..d6670bcf83 100644 --- a/packages/docgen/package.json +++ b/packages/docgen/package.json @@ -28,7 +28,7 @@ "dependencies": { "chalk": "^4.0.0", "commander": "^6.1.0", - "fs-extra": "^9.0.0", + "fs-extra": "^10.0.0", "github-slugger": "^1.3.0", "ts-node": "^10.0.0", "typescript": "^4.0.3" diff --git a/packages/e2e-test/package.json b/packages/e2e-test/package.json index 304da9816b..b4e61a7cc1 100644 --- a/packages/e2e-test/package.json +++ b/packages/e2e-test/package.json @@ -30,7 +30,7 @@ "chalk": "^4.0.0", "commander": "^6.1.0", "cross-fetch": "^3.0.6", - "fs-extra": "^9.0.0", + "fs-extra": "^10.0.0", "handlebars": "^4.7.3", "pgtools": "^0.3.0", "tree-kill": "^1.2.2", diff --git a/packages/integration/src/bitbucket/core.test.ts b/packages/integration/src/bitbucket/core.test.ts index 62f81179ee..0de1d5aaab 100644 --- a/packages/integration/src/bitbucket/core.test.ts +++ b/packages/integration/src/bitbucket/core.test.ts @@ -192,7 +192,7 @@ describe('bitbucket core', () => { config, ); expect(result).toEqual( - 'https://bitbucket.org/backstage/mock/get/master.tgz', + 'https://bitbucket.org/backstage/mock/get/master.tar.gz', ); }); }); diff --git a/packages/integration/src/bitbucket/core.ts b/packages/integration/src/bitbucket/core.ts index 0a1b247f10..f8f8f56fa8 100644 --- a/packages/integration/src/bitbucket/core.ts +++ b/packages/integration/src/bitbucket/core.ts @@ -98,7 +98,7 @@ export async function getBitbucketDownloadUrl( // /docs/index.md will download the docs folder and everything below it const path = filepath ? `&path=${encodeURIComponent(filepath)}` : ''; const archiveUrl = isHosted - ? `${protocol}://${resource}/${project}/${repoName}/get/${branch}.tgz` + ? `${protocol}://${resource}/${project}/${repoName}/get/${branch}.tar.gz` : `${config.apiBaseUrl}/projects/${project}/repos/${repoName}/archive?format=tgz&at=${branch}&prefix=${project}-${repoName}${path}`; return archiveUrl; diff --git a/packages/integration/src/helpers.ts b/packages/integration/src/helpers.ts index 20c282c5a3..b1e74035f1 100644 --- a/packages/integration/src/helpers.ts +++ b/packages/integration/src/helpers.ts @@ -44,8 +44,12 @@ export function basicIntegrations( return integrations; }, byUrl(url: string | URL): T | undefined { - const parsed = typeof url === 'string' ? new URL(url) : url; - return integrations.find(i => getHost(i) === parsed.hostname); + try { + const parsed = typeof url === 'string' ? new URL(url) : url; + return integrations.find(i => getHost(i) === parsed.hostname); + } catch { + return undefined; + } }, byHost(host: string): T | undefined { return integrations.find(i => getHost(i) === host); diff --git a/packages/techdocs-common/package.json b/packages/techdocs-common/package.json index 59b18d8e27..7cb0e37174 100644 --- a/packages/techdocs-common/package.json +++ b/packages/techdocs-common/package.json @@ -48,7 +48,7 @@ "aws-sdk": "^2.840.0", "cross-fetch": "^3.0.6", "express": "^4.17.1", - "fs-extra": "^9.0.1", + "fs-extra": "^10.0.0", "git-url-parse": "^11.4.4", "js-yaml": "^4.0.0", "json5": "^2.1.3", diff --git a/packages/techdocs-common/src/stages/publish/openStackSwift.ts b/packages/techdocs-common/src/stages/publish/openStackSwift.ts index 18944b5605..6be8539aa9 100644 --- a/packages/techdocs-common/src/stages/publish/openStackSwift.ts +++ b/packages/techdocs-common/src/stages/publish/openStackSwift.ts @@ -156,7 +156,7 @@ export class OpenStackSwiftPublish implements PublisherBase { const uploadFile = limiter( () => new Promise((res, rej) => { - const readStream = fs.createReadStream(filePath, 'utf8'); + const readStream = fs.createReadStream(filePath); const writeStream = this.storageClient.upload(params); diff --git a/plugins/app-backend/package.json b/plugins/app-backend/package.json index 3b79162624..384d44d82a 100644 --- a/plugins/app-backend/package.json +++ b/plugins/app-backend/package.json @@ -35,7 +35,7 @@ "@types/express": "^4.17.6", "express": "^4.17.1", "express-promise-router": "^4.1.0", - "fs-extra": "^9.0.0", + "fs-extra": "^10.0.0", "winston": "^3.2.1", "yn": "^4.0.0" }, diff --git a/plugins/auth-backend/package.json b/plugins/auth-backend/package.json index 98c9d8b62f..085590c8fa 100644 --- a/plugins/auth-backend/package.json +++ b/plugins/auth-backend/package.json @@ -44,7 +44,7 @@ "express": "^4.17.1", "express-promise-router": "^4.1.0", "express-session": "^1.17.1", - "fs-extra": "^9.0.0", + "fs-extra": "^10.0.0", "got": "^11.5.2", "helmet": "^4.0.0", "jose": "^1.27.1", diff --git a/plugins/catalog-backend-module-ldap/api-report.md b/plugins/catalog-backend-module-ldap/api-report.md index d5e9d731ea..55405cab41 100644 --- a/plugins/catalog-backend-module-ldap/api-report.md +++ b/plugins/catalog-backend-module-ldap/api-report.md @@ -16,6 +16,15 @@ import { SearchEntry } from 'ldapjs'; import { SearchOptions } from 'ldapjs'; import { UserEntity } from '@backstage/catalog-model'; +// @public (undocumented) +export function defaultGroupTransformer(vendor: LdapVendor, config: GroupConfig, entry: SearchEntry): Promise; + +// @public (undocumented) +export function defaultUserTransformer(vendor: LdapVendor, config: UserConfig, entry: SearchEntry): Promise; + +// @public +export type GroupTransformer = (vendor: LdapVendor, config: GroupConfig, group: SearchEntry) => Promise; + // @public export const LDAP_DN_ANNOTATION = "backstage.io/ldap-dn"; @@ -40,14 +49,18 @@ export class LdapOrgReaderProcessor implements CatalogProcessor { constructor(options: { providers: LdapProviderConfig[]; logger: Logger; + groupTransformer?: GroupTransformer; + userTransformer?: UserTransformer; }); // (undocumented) static fromConfig(config: Config, options: { logger: Logger; + groupTransformer?: GroupTransformer; + userTransformer?: UserTransformer; }): LdapOrgReaderProcessor; // (undocumented) readLocation(location: LocationSpec, _optional: boolean, emit: CatalogProcessorEmit): Promise; -} + } // @public export type LdapProviderConfig = { @@ -57,15 +70,25 @@ export type LdapProviderConfig = { groups: GroupConfig; }; +// @public +export function mapStringAttr(entry: SearchEntry, vendor: LdapVendor, attributeName: string | undefined, setter: (value: string) => void): void; + // @public export function readLdapConfig(config: Config): LdapProviderConfig[]; // @public -export function readLdapOrg(client: LdapClient, userConfig: UserConfig, groupConfig: GroupConfig): Promise<{ +export function readLdapOrg(client: LdapClient, userConfig: UserConfig, groupConfig: GroupConfig, options: { + groupTransformer?: GroupTransformer; + userTransformer?: UserTransformer; + logger: Logger; +}): Promise<{ users: UserEntity[]; groups: GroupEntity[]; }>; +// @public +export type UserTransformer = (vendor: LdapVendor, config: UserConfig, user: SearchEntry) => Promise; + // (No @packageDocumentation comment for this package) diff --git a/plugins/catalog-backend-module-ldap/src/ldap/index.ts b/plugins/catalog-backend-module-ldap/src/ldap/index.ts index 194a75ac60..a9f127a3a7 100644 --- a/plugins/catalog-backend-module-ldap/src/ldap/index.ts +++ b/plugins/catalog-backend-module-ldap/src/ldap/index.ts @@ -15,6 +15,7 @@ */ export { LdapClient } from './client'; +export { mapStringAttr } from './util'; export { readLdapConfig } from './config'; export type { LdapProviderConfig } from './config'; export { @@ -22,4 +23,9 @@ export { LDAP_RDN_ANNOTATION, LDAP_UUID_ANNOTATION, } from './constants'; -export { readLdapOrg } from './read'; +export { + defaultGroupTransformer, + defaultUserTransformer, + readLdapOrg, +} from './read'; +export type { GroupTransformer, UserTransformer } from './types'; diff --git a/plugins/catalog-backend-module-ldap/src/ldap/read.ts b/plugins/catalog-backend-module-ldap/src/ldap/read.ts index a662b36d87..f3ee09d32d 100644 --- a/plugins/catalog-backend-module-ldap/src/ldap/read.ts +++ b/plugins/catalog-backend-module-ldap/src/ldap/read.ts @@ -26,74 +26,97 @@ import { LDAP_UUID_ANNOTATION, } from './constants'; import { LdapVendor } from './vendors'; +import { Logger } from 'winston'; +import { GroupTransformer, UserTransformer } from './types'; +import { mapStringAttr } from './util'; + +export async function defaultUserTransformer( + vendor: LdapVendor, + config: UserConfig, + entry: SearchEntry, +): Promise { + const { set, map } = config; + + const entity: UserEntity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + name: '', + annotations: {}, + }, + spec: { + profile: {}, + memberOf: [], + }, + }; + + if (set) { + for (const [path, value] of Object.entries(set)) { + lodashSet(entity, path, value); + } + } + + mapStringAttr(entry, vendor, map.name, v => { + entity.metadata.name = v; + }); + mapStringAttr(entry, vendor, map.description, v => { + entity.metadata.description = v; + }); + mapStringAttr(entry, vendor, map.rdn, v => { + entity.metadata.annotations![LDAP_RDN_ANNOTATION] = v; + }); + mapStringAttr(entry, vendor, vendor.uuidAttributeName, v => { + entity.metadata.annotations![LDAP_UUID_ANNOTATION] = v; + }); + mapStringAttr(entry, vendor, vendor.dnAttributeName, v => { + entity.metadata.annotations![LDAP_DN_ANNOTATION] = v; + }); + mapStringAttr(entry, vendor, map.displayName, v => { + entity.spec.profile!.displayName = v; + }); + mapStringAttr(entry, vendor, map.email, v => { + entity.spec.profile!.email = v; + }); + mapStringAttr(entry, vendor, map.picture, v => { + entity.spec.profile!.picture = v; + }); + + return entity; +} /** * Reads users out of an LDAP provider. * * @param client The LDAP client * @param config The user data configuration + * @param opts */ export async function readLdapUsers( client: LdapClient, config: UserConfig, + opts?: { transformer?: UserTransformer }, ): Promise<{ users: UserEntity[]; // With all relations empty userMemberOf: Map>; // DN -> DN or UUID of groups }> { - const { dn, options, set, map } = config; + const { dn, options, map } = config; const vendor = await client.getVendor(); - const entries = await client.search(dn, options); - const entities: UserEntity[] = []; const userMemberOf: Map> = new Map(); - for (const entry of entries) { - const entity: UserEntity = { - apiVersion: 'backstage.io/v1alpha1', - kind: 'User', - metadata: { - name: '', - annotations: {}, - }, - spec: { - profile: {}, - memberOf: [], - }, - }; + const transformer = opts?.transformer ?? defaultUserTransformer; - if (set) { - for (const [path, value] of Object.entries(set)) { - lodashSet(entity, path, value); - } + const entries = await client.search(dn, options); + + for (const user of entries) { + const entity = await transformer(vendor, config, user); + + if (!entity) { + continue; } - mapStringAttr(entry, vendor, map.name, v => { - entity.metadata.name = v; - }); - mapStringAttr(entry, vendor, map.description, v => { - entity.metadata.description = v; - }); - mapStringAttr(entry, vendor, map.rdn, v => { - entity.metadata.annotations![LDAP_RDN_ANNOTATION] = v; - }); - mapStringAttr(entry, vendor, vendor.uuidAttributeName, v => { - entity.metadata.annotations![LDAP_UUID_ANNOTATION] = v; - }); - mapStringAttr(entry, vendor, vendor.dnAttributeName, v => { - entity.metadata.annotations![LDAP_DN_ANNOTATION] = v; - }); - mapStringAttr(entry, vendor, map.displayName, v => { - entity.spec.profile!.displayName = v; - }); - mapStringAttr(entry, vendor, map.email, v => { - entity.spec.profile!.email = v; - }); - mapStringAttr(entry, vendor, map.picture, v => { - entity.spec.profile!.picture = v; - }); - - mapReferencesAttr(entry, vendor, map.memberOf, (myDn, vs) => { + mapReferencesAttr(user, vendor, map.memberOf, (myDn, vs) => { ensureItems(userMemberOf, myDn, vs); }); @@ -103,82 +126,103 @@ export async function readLdapUsers( return { users: entities, userMemberOf }; } +export async function defaultGroupTransformer( + vendor: LdapVendor, + config: GroupConfig, + entry: SearchEntry, +): Promise { + const { set, map } = config; + const entity: GroupEntity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + name: '', + annotations: {}, + }, + spec: { + type: 'unknown', + profile: {}, + children: [], + }, + }; + + if (set) { + for (const [path, value] of Object.entries(set)) { + lodashSet(entity, path, value); + } + } + + mapStringAttr(entry, vendor, map.name, v => { + entity.metadata.name = v; + }); + mapStringAttr(entry, vendor, map.description, v => { + entity.metadata.description = v; + }); + mapStringAttr(entry, vendor, map.rdn, v => { + entity.metadata.annotations![LDAP_RDN_ANNOTATION] = v; + }); + mapStringAttr(entry, vendor, vendor.uuidAttributeName, v => { + entity.metadata.annotations![LDAP_UUID_ANNOTATION] = v; + }); + mapStringAttr(entry, vendor, vendor.dnAttributeName, v => { + entity.metadata.annotations![LDAP_DN_ANNOTATION] = v; + }); + mapStringAttr(entry, vendor, map.type, v => { + entity.spec.type = v; + }); + mapStringAttr(entry, vendor, map.displayName, v => { + entity.spec.profile!.displayName = v; + }); + mapStringAttr(entry, vendor, map.email, v => { + entity.spec.profile!.email = v; + }); + mapStringAttr(entry, vendor, map.picture, v => { + entity.spec.profile!.picture = v; + }); + + return entity; +} + /** * Reads groups out of an LDAP provider. * * @param client The LDAP client * @param config The group data configuration + * @param opts */ export async function readLdapGroups( client: LdapClient, config: GroupConfig, + opts?: { + transformer?: GroupTransformer; + }, ): Promise<{ groups: GroupEntity[]; // With all relations empty groupMemberOf: Map>; // DN -> DN or UUID of groups groupMember: Map>; // DN -> DN or UUID of groups & users }> { - const { dn, options, set, map } = config; - const vendor = await client.getVendor(); - - const entries = await client.search(dn, options); - const groups: GroupEntity[] = []; const groupMemberOf: Map> = new Map(); const groupMember: Map> = new Map(); - for (const entry of entries) { - const entity: GroupEntity = { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Group', - metadata: { - name: '', - annotations: {}, - }, - spec: { - type: 'unknown', - profile: {}, - children: [], - }, - }; + const { dn, map, options } = config; + const vendor = await client.getVendor(); - if (set) { - for (const [path, value] of Object.entries(set)) { - lodashSet(entity, path, value); - } + const transformer = opts?.transformer ?? defaultGroupTransformer; + + const entries = await client.search(dn, options); + + for (const group of entries) { + const entity = await transformer(vendor, config, group); + + if (!entity) { + continue; } - mapStringAttr(entry, vendor, map.name, v => { - entity.metadata.name = v; - }); - mapStringAttr(entry, vendor, map.description, v => { - entity.metadata.description = v; - }); - mapStringAttr(entry, vendor, map.rdn, v => { - entity.metadata.annotations![LDAP_RDN_ANNOTATION] = v; - }); - mapStringAttr(entry, vendor, vendor.uuidAttributeName, v => { - entity.metadata.annotations![LDAP_UUID_ANNOTATION] = v; - }); - mapStringAttr(entry, vendor, vendor.dnAttributeName, v => { - entity.metadata.annotations![LDAP_DN_ANNOTATION] = v; - }); - mapStringAttr(entry, vendor, map.type, v => { - entity.spec.type = v; - }); - mapStringAttr(entry, vendor, map.displayName, v => { - entity.spec.profile!.displayName = v; - }); - mapStringAttr(entry, vendor, map.email, v => { - entity.spec.profile!.email = v; - }); - mapStringAttr(entry, vendor, map.picture, v => { - entity.spec.profile!.picture = v; - }); - - mapReferencesAttr(entry, vendor, map.memberOf, (myDn, vs) => { + mapReferencesAttr(group, vendor, map.memberOf, (myDn, vs) => { ensureItems(groupMemberOf, myDn, vs); }); - mapReferencesAttr(entry, vendor, map.members, (myDn, vs) => { + mapReferencesAttr(group, vendor, map.members, (myDn, vs) => { ensureItems(groupMember, myDn, vs); }); @@ -199,22 +243,30 @@ export async function readLdapGroups( * with all relations etc filled in. * * @param client The LDAP client - * @param logger A logger instance * @param userConfig The user data configuration * @param groupConfig The group data configuration + * @param options */ export async function readLdapOrg( client: LdapClient, userConfig: UserConfig, groupConfig: GroupConfig, + options: { + groupTransformer?: GroupTransformer; + userTransformer?: UserTransformer; + logger: Logger; + }, ): Promise<{ users: UserEntity[]; groups: GroupEntity[]; }> { - const { users, userMemberOf } = await readLdapUsers(client, userConfig); + const { users, userMemberOf } = await readLdapUsers(client, userConfig, { + transformer: options?.userTransformer, + }); const { groups, groupMemberOf, groupMember } = await readLdapGroups( client, groupConfig, + { transformer: options?.groupTransformer }, ); resolveRelations(groups, users, userMemberOf, groupMemberOf, groupMember); @@ -228,21 +280,6 @@ export async function readLdapOrg( // Helpers // -// Maps a single-valued attribute to a consumer -function mapStringAttr( - entry: SearchEntry, - vendor: LdapVendor, - attributeName: string | undefined, - setter: (value: string) => void, -) { - if (attributeName) { - const values = vendor.decodeStringAttribute(entry, attributeName); - if (values && values.length === 1) { - setter(values[0]); - } - } -} - // Maps a multi-valued attribute of references to other objects, to a consumer function mapReferencesAttr( entry: SearchEntry, diff --git a/plugins/catalog-backend-module-ldap/src/ldap/types.ts b/plugins/catalog-backend-module-ldap/src/ldap/types.ts new file mode 100644 index 0000000000..e32e4c5b42 --- /dev/null +++ b/plugins/catalog-backend-module-ldap/src/ldap/types.ts @@ -0,0 +1,47 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { GroupEntity, UserEntity } from '@backstage/catalog-model'; +import { SearchEntry } from 'ldapjs'; +import { LdapVendor } from './vendors'; +import { GroupConfig, UserConfig } from './config'; + +/** + * Customize the ingested User entity + * + * @param vendor The LDAP vendor that can be used to find and decode vendor specific attributes + * @param config The User specific config used by the default transformer. + * @param user The found LDAP entry in its source format. This is the entry that you want to transform + * @return A `UserEntity` or `undefined` if you want to ignore the found user for being ingested by the catalog + */ +export type UserTransformer = ( + vendor: LdapVendor, + config: UserConfig, + user: SearchEntry, +) => Promise; + +/** + * Customize the ingested Group entity + * + * @param vendor The LDAP vendor that can be used to find and decode vendor specific attributes + * @param config The Group specific config used by the default transformer. + * @param group The found LDAP entry in its source format. This is the entry that you want to transform + * @return A `GroupEntity` or `undefined` if you want to ignore the found group for being ingested by the catalog + */ +export type GroupTransformer = ( + vendor: LdapVendor, + config: GroupConfig, + group: SearchEntry, +) => Promise; diff --git a/plugins/catalog-backend-module-ldap/src/ldap/util.ts b/plugins/catalog-backend-module-ldap/src/ldap/util.ts index 74dd84cdc9..21ee40f52c 100644 --- a/plugins/catalog-backend-module-ldap/src/ldap/util.ts +++ b/plugins/catalog-backend-module-ldap/src/ldap/util.ts @@ -14,7 +14,8 @@ * limitations under the License. */ -import { Error as LDAPError } from 'ldapjs'; +import { Error as LDAPError, SearchEntry } from 'ldapjs'; +import { LdapVendor } from './vendors'; /** * Builds a string form of an LDAP Error structure. @@ -24,3 +25,25 @@ import { Error as LDAPError } from 'ldapjs'; export function errorString(error: LDAPError) { return `${error.code} ${error.name}: ${error.message}`; } + +/** + * Maps a single-valued attribute to a consumer + * + * @param entry The LDAP source entry + * @param vendor The LDAP vendor + * @param attributeName The source attribute to map. If the attribute is undefined the mapping will be silently ignored. + * @param setter The function to be called with the decoded attribute from the source entry + */ +export function mapStringAttr( + entry: SearchEntry, + vendor: LdapVendor, + attributeName: string | undefined, + setter: (value: string) => void, +) { + if (attributeName) { + const values = vendor.decodeStringAttribute(entry, attributeName); + if (values && values.length === 1) { + setter(values[0]); + } + } +} diff --git a/plugins/catalog-backend-module-ldap/src/processors/LdapOrgReaderProcessor.ts b/plugins/catalog-backend-module-ldap/src/processors/LdapOrgReaderProcessor.ts index 79c0ed6964..f87afc38b9 100644 --- a/plugins/catalog-backend-module-ldap/src/processors/LdapOrgReaderProcessor.ts +++ b/plugins/catalog-backend-module-ldap/src/processors/LdapOrgReaderProcessor.ts @@ -18,10 +18,12 @@ import { LocationSpec } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; import { Logger } from 'winston'; import { + GroupTransformer, LdapClient, LdapProviderConfig, readLdapConfig, readLdapOrg, + UserTransformer, } from '../ldap'; import { CatalogProcessor, @@ -35,8 +37,17 @@ import { export class LdapOrgReaderProcessor implements CatalogProcessor { private readonly providers: LdapProviderConfig[]; private readonly logger: Logger; + private readonly groupTransformer?: GroupTransformer; + private readonly userTransformer?: UserTransformer; - static fromConfig(config: Config, options: { logger: Logger }) { + static fromConfig( + config: Config, + options: { + logger: Logger; + groupTransformer?: GroupTransformer; + userTransformer?: UserTransformer; + }, + ) { const c = config.getOptionalConfig('catalog.processors.ldapOrg'); return new LdapOrgReaderProcessor({ ...options, @@ -44,9 +55,16 @@ export class LdapOrgReaderProcessor implements CatalogProcessor { }); } - constructor(options: { providers: LdapProviderConfig[]; logger: Logger }) { + constructor(options: { + providers: LdapProviderConfig[]; + logger: Logger; + groupTransformer?: GroupTransformer; + userTransformer?: UserTransformer; + }) { this.providers = options.providers; this.logger = options.logger; + this.groupTransformer = options.groupTransformer; + this.userTransformer = options.userTransformer; } async readLocation( @@ -81,6 +99,11 @@ export class LdapOrgReaderProcessor implements CatalogProcessor { client, provider.users, provider.groups, + { + groupTransformer: this.groupTransformer, + userTransformer: this.userTransformer, + logger: this.logger, + }, ); const duration = ((Date.now() - startTimestamp) / 1000).toFixed(1); diff --git a/plugins/catalog-backend/package.json b/plugins/catalog-backend/package.json index 3a0464bff6..2a8588c964 100644 --- a/plugins/catalog-backend/package.json +++ b/plugins/catalog-backend/package.json @@ -46,7 +46,7 @@ "express": "^4.17.1", "express-promise-router": "^4.1.0", "fast-json-stable-stringify": "^2.1.0", - "fs-extra": "^9.0.0", + "fs-extra": "^10.0.0", "git-url-parse": "^11.4.4", "glob": "^7.1.6", "knex": "^0.95.1", diff --git a/plugins/catalog-backend/src/ingestion/processors/PlaceholderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/PlaceholderProcessor.ts index 8c06ef419e..1b015d7d93 100644 --- a/plugins/catalog-backend/src/ingestion/processors/PlaceholderProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/PlaceholderProcessor.ts @@ -94,12 +94,21 @@ export class PlaceholderProcessor implements CatalogProcessor { return [data, false]; } + const read = async (url: string): Promise => { + if (this.options.reader.readUrl) { + const response = await this.options.reader.readUrl(url); + const buffer = await response.buffer(); + return buffer; + } + return this.options.reader.read(url); + }; + return [ await resolver({ key: resolverKey, value: resolverValue, baseUrl: location.target, - read: this.options.reader.read.bind(this.options.reader), + read, }), true, ]; diff --git a/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.ts index aff126414f..da7bee12d8 100644 --- a/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.ts @@ -101,7 +101,12 @@ export class UrlReaderProcessor implements CatalogProcessor { return Promise.all(output); } - // Otherwise do a plain read + // Otherwise do a plain read, prioritizing readUrl if available + if (this.options.reader.readUrl) { + const data = await this.options.reader.readUrl(location); + return [{ url: location, data: await data.buffer() }]; + } + const data = await this.options.reader.read(location); return [{ url: location, data }]; } diff --git a/plugins/catalog-backend/src/ingestion/processors/codeowners/read.ts b/plugins/catalog-backend/src/ingestion/processors/codeowners/read.ts index ae4255555a..c6b89efcc9 100644 --- a/plugins/catalog-backend/src/ingestion/processors/codeowners/read.ts +++ b/plugins/catalog-backend/src/ingestion/processors/codeowners/read.ts @@ -28,6 +28,12 @@ export async function readCodeOwners( ): Promise { const readOwnerLocation = async (path: string): Promise => { const url = `${sourceUrl}${path}`; + + if (reader.readUrl) { + const data = await reader.readUrl(url); + const buffer = await data.buffer(); + return buffer.toString(); + } const data = await reader.read(url); return data.toString(); }; diff --git a/plugins/catalog-import/src/api/CatalogImportClient.test.ts b/plugins/catalog-import/src/api/CatalogImportClient.test.ts index 8ab8f75db1..e2c3f2dc47 100644 --- a/plugins/catalog-import/src/api/CatalogImportClient.test.ts +++ b/plugins/catalog-import/src/api/CatalogImportClient.test.ts @@ -46,29 +46,19 @@ jest.doMock('@octokit/rest', () => { return { Octokit }; }); -// Mock the value to control which integrations are activated -jest.mock('./GitHub', () => ({ - getGithubIntegrationConfig: jest.fn(), -})); - -import { - GitHubIntegrationConfig, - ScmIntegrations, -} from '@backstage/integration'; +import { ScmIntegrations } from '@backstage/integration'; import { catalogApiRef } from '@backstage/plugin-catalog-react'; import { msw } from '@backstage/test-utils'; import { Octokit } from '@octokit/rest'; import { rest } from 'msw'; import { setupServer } from 'msw/node'; import { CatalogImportClient } from './CatalogImportClient'; -import { getGithubIntegrationConfig } from './GitHub'; import { ConfigReader, UrlPatternDiscovery } from '@backstage/core-app-api'; import { OAuthApi } from '@backstage/core-plugin-api'; -const server = setupServer(); - describe('CatalogImportClient', () => { + const server = setupServer(); msw.setupDefaultHandlers(server); const mockBaseUrl = 'http://backstage:9191/api/catalog'; @@ -92,7 +82,19 @@ describe('CatalogImportClient', () => { }, }; - const scmIntegrationsApi = ScmIntegrations.fromConfig(new ConfigReader({})); + const scmIntegrationsApi = ScmIntegrations.fromConfig( + new ConfigReader({ + integrations: { + github: [{ host: 'example.com' }], + gitlab: [ + { + host: 'registered-but-not-github.com', + apiBaseUrl: 'https://registered-but-not-github.com', + }, + ], + }, + }), + ); const catalogApi: jest.Mocked = { getEntities: jest.fn(), @@ -106,7 +108,6 @@ describe('CatalogImportClient', () => { }; let catalogImportClient: CatalogImportClient; - let getGithubIntegrationConfigFn: jest.Mock; beforeEach(() => { catalogImportClient = new CatalogImportClient({ @@ -116,9 +117,6 @@ describe('CatalogImportClient', () => { identityApi, catalogApi, }); - - getGithubIntegrationConfigFn = getGithubIntegrationConfig as jest.Mock; - getGithubIntegrationConfigFn.mockReset(); }); describe('analyzeUrl', () => { @@ -169,10 +167,22 @@ describe('CatalogImportClient', () => { }); }); - it('should ignore missing github integration', async () => { + it('should reject for integrations that are not github ones', async () => { await expect( catalogImportClient.analyzeUrl( - 'https://github.com/backstage/backstage', + 'https://registered-but-not-github.com/backstage/backstage', + ), + ).rejects.toThrow( + new Error( + 'The registered-but-not-github.com integration only supports full URLs to catalog-info.yaml files. Did you try to pass in the URL of a directory instead?', + ), + ); + }); + + it('should reject when unable to match with any integration', async () => { + await expect( + catalogImportClient.analyzeUrl( + 'https://not-registered-as-integration.com/foo/bar', ), ).rejects.toThrow( new Error( @@ -182,12 +192,6 @@ describe('CatalogImportClient', () => { }); it('should find locations from github', async () => { - getGithubIntegrationConfigFn.mockReturnValue({ - repo: 'backstage', - owner: 'backstage', - githubIntegrationConfig: {} as GitHubIntegrationConfig, - }); - ((new Octokit().search.code as any) as jest.Mock).mockResolvedValueOnce({ data: { total_count: 2, @@ -244,12 +248,6 @@ describe('CatalogImportClient', () => { }); it('should find repository from github', async () => { - getGithubIntegrationConfigFn.mockReturnValue({ - repo: 'backstage', - owner: 'backstage', - githubIntegrationConfig: {} as GitHubIntegrationConfig, - }); - ((new Octokit().search.code as any) as jest.Mock).mockResolvedValueOnce({ data: { total_count: 0, items: [] }, }); @@ -300,14 +298,6 @@ describe('CatalogImportClient', () => { describe('submitPullRequest', () => { it('should create GitHub pull request', async () => { - getGithubIntegrationConfigFn.mockReturnValue({ - repo: 'backstage', - owner: 'backstage', - githubIntegrationConfig: { - host: 'github.com', - } as GitHubIntegrationConfig, - }); - await expect( catalogImportClient.submitPullRequest({ repositoryUrl: 'https://github.com/backstage/backstage', diff --git a/plugins/catalog-import/src/api/CatalogImportClient.ts b/plugins/catalog-import/src/api/CatalogImportClient.ts index f53570617e..aa29c95247 100644 --- a/plugins/catalog-import/src/api/CatalogImportClient.ts +++ b/plugins/catalog-import/src/api/CatalogImportClient.ts @@ -77,6 +77,12 @@ export class CatalogImportClient implements CatalogImportApi { const ghConfig = getGithubIntegrationConfig(this.scmIntegrationsApi, url); if (!ghConfig) { + const other = this.scmIntegrationsApi.byUrl(url); + if (other) { + throw new Error( + `The ${other.title} integration only supports full URLs to catalog-info.yaml files. Did you try to pass in the URL of a directory instead?`, + ); + } throw new Error( 'This URL was not recognized as a valid GitHub URL because there was no configured integration that matched the given host name. You could try to paste the full URL to a catalog-info.yaml file instead.', ); diff --git a/plugins/catalog/src/components/CatalogPage/CatalogPage.tsx b/plugins/catalog/src/components/CatalogPage/CatalogPage.tsx index ba19d6036c..089e61f0fb 100644 --- a/plugins/catalog/src/components/CatalogPage/CatalogPage.tsx +++ b/plugins/catalog/src/components/CatalogPage/CatalogPage.tsx @@ -15,7 +15,7 @@ */ import React from 'react'; -import { makeStyles } from '@material-ui/core'; +import { Grid } from '@material-ui/core'; import { EntityKindPicker, EntityLifecyclePicker, @@ -39,18 +39,6 @@ import { TableProps, } from '@backstage/core-components'; -const useStyles = makeStyles(theme => ({ - contentWrapper: { - display: 'grid', - gridTemplateAreas: "'filters' 'table'", - gridTemplateColumns: '250px 1fr', - gridColumnGap: theme.spacing(2), - }, - buttonSpacing: { - marginLeft: theme.spacing(2), - }, -})); - export type CatalogPageProps = { initiallySelectedFilter?: UserListFilterKind; columns?: TableColumn[]; @@ -61,30 +49,36 @@ export const CatalogPage = ({ initiallySelectedFilter = 'owned', columns, actions, -}: CatalogPageProps) => { - const styles = useStyles(); - - return ( - - - - - All your software catalog entities - -
- -
-
+}: CatalogPageProps) => ( + + + + + All your software catalog entities + + + + + + + + + + + + + + + + + + - -
- - - ); -}; + + + + + +); diff --git a/plugins/explore/src/components/ToolCard/ToolCard.tsx b/plugins/explore/src/components/ToolCard/ToolCard.tsx index 2baa0cffda..904db0c00b 100644 --- a/plugins/explore/src/components/ToolCard/ToolCard.tsx +++ b/plugins/explore/src/components/ToolCard/ToolCard.tsx @@ -18,7 +18,6 @@ import { ExploreTool } from '@backstage/plugin-explore-react'; import { BackstageTheme } from '@backstage/theme'; import { Box, - Button, Card, CardActions, CardContent, @@ -27,6 +26,7 @@ import { makeStyles, Typography, } from '@material-ui/core'; +import { Button } from '@backstage/core-components'; import classNames from 'classnames'; import React from 'react'; @@ -97,7 +97,7 @@ export const ToolCard = ({ card, objectFit }: Props) => { )} - diff --git a/plugins/git-release-manager/src/features/Features.test.tsx b/plugins/git-release-manager/src/features/Features.test.tsx index ac3004f690..b5404a21d9 100644 --- a/plugins/git-release-manager/src/features/Features.test.tsx +++ b/plugins/git-release-manager/src/features/Features.test.tsx @@ -69,6 +69,7 @@ describe('Features', () => { diff --git a/plugins/ilert/package.json b/plugins/ilert/package.json index 1664831179..521f379d64 100644 --- a/plugins/ilert/package.json +++ b/plugins/ilert/package.json @@ -26,7 +26,7 @@ "@backstage/errors": "^0.1.1", "@backstage/plugin-catalog-react": "^0.2.4", "@backstage/theme": "^0.2.8", - "@date-io/luxon": "1.x", + "@date-io/luxon": "2.x", "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", "@material-ui/lab": "4.0.0-alpha.45", diff --git a/plugins/kubernetes-backend/package.json b/plugins/kubernetes-backend/package.json index 39297dd109..72f576616e 100644 --- a/plugins/kubernetes-backend/package.json +++ b/plugins/kubernetes-backend/package.json @@ -44,7 +44,7 @@ "cors": "^2.8.5", "express": "^4.17.1", "express-promise-router": "^4.1.0", - "fs-extra": "^9.0.0", + "fs-extra": "^10.0.0", "helmet": "^4.0.0", "lodash": "^4.17.15", "morgan": "^1.10.0", diff --git a/plugins/org/src/components/Cards/Group/GroupProfile/GroupProfileCard.tsx b/plugins/org/src/components/Cards/Group/GroupProfile/GroupProfileCard.tsx index 853dbc53e8..b44fbe6c5c 100644 --- a/plugins/org/src/components/Cards/Group/GroupProfile/GroupProfileCard.tsx +++ b/plugins/org/src/components/Cards/Group/GroupProfile/GroupProfileCard.tsx @@ -23,23 +23,30 @@ import { EntityRefLinks, getEntityRelations, useEntity, + getEntityMetadataEditUrl, } from '@backstage/plugin-catalog-react'; import { Box, Grid, - Link, List, ListItem, ListItemIcon, ListItemText, Tooltip, + IconButton, } from '@material-ui/core'; import AccountTreeIcon from '@material-ui/icons/AccountTree'; import EmailIcon from '@material-ui/icons/Email'; import GroupIcon from '@material-ui/icons/Group'; +import EditIcon from '@material-ui/icons/Edit'; import Alert from '@material-ui/lab/Alert'; import React from 'react'; -import { Avatar, InfoCard, InfoCardVariants } from '@backstage/core-components'; +import { + Avatar, + InfoCard, + InfoCardVariants, + Link, +} from '@backstage/core-components'; const CardTitle = ({ title }: { title: string }) => ( @@ -72,14 +79,31 @@ export const GroupProfileCard = ({ kind: 'group', }); + const entityMetadataEditUrl = getEntityMetadataEditUrl(group); + const displayName = profile?.displayName ?? name; - const emailHref = profile?.email ? `mailto:${profile.email}` : undefined; + const emailHref = profile?.email ? `mailto:${profile.email}` : '#'; + const infoCardAction = entityMetadataEditUrl ? ( + + + + ) : ( + + + + ); return ( } subheader={description} variant={variant} + action={infoCardAction} > @@ -95,7 +119,7 @@ export const GroupProfileCard = ({ - {profile.email} + {profile.email} )} diff --git a/plugins/pagerduty/README.md b/plugins/pagerduty/README.md index e1ec7f89c7..bdafeb984a 100644 --- a/plugins/pagerduty/README.md +++ b/plugins/pagerduty/README.md @@ -8,11 +8,21 @@ There is also an easy way to trigger an alarm directly to the person who is curr This plugin requires that entities are annotated with an [integration key](https://support.pagerduty.com/docs/services-and-integrations#add-integrations-to-an-existing-service). See more further down in this document. -This plugin provides: +## Features -- A list of incidents -- A way to trigger an alarm to the person on-call -- Information details about the person on-call +### View any open incidents + +![PagerDuty plugin showing no incidents and the on-call rotation](doc/pd1.png) + +### Email link, and view contact information for staff on call + +![PagerDuty plugin showing on-call rotation contact information](doc/pd2.png) + +### Trigger an incident for a service + +![PagerDuty plugin popup modal for creating an incident](doc/pd3.png) + +![PagerDuty plugin showing an active incident](doc/pd4.png) ## Setup instructions diff --git a/plugins/pagerduty/doc/pd1.png b/plugins/pagerduty/doc/pd1.png new file mode 100644 index 0000000000..8242792b5a Binary files /dev/null and b/plugins/pagerduty/doc/pd1.png differ diff --git a/plugins/pagerduty/doc/pd2.png b/plugins/pagerduty/doc/pd2.png new file mode 100644 index 0000000000..816abd5e66 Binary files /dev/null and b/plugins/pagerduty/doc/pd2.png differ diff --git a/plugins/pagerduty/doc/pd3.png b/plugins/pagerduty/doc/pd3.png new file mode 100644 index 0000000000..89d7f1820d Binary files /dev/null and b/plugins/pagerduty/doc/pd3.png differ diff --git a/plugins/pagerduty/doc/pd4.png b/plugins/pagerduty/doc/pd4.png new file mode 100644 index 0000000000..70584bc59e Binary files /dev/null and b/plugins/pagerduty/doc/pd4.png differ diff --git a/plugins/rollbar-backend/package.json b/plugins/rollbar-backend/package.json index e14f6f5a6c..ad3c04698d 100644 --- a/plugins/rollbar-backend/package.json +++ b/plugins/rollbar-backend/package.json @@ -39,7 +39,7 @@ "cors": "^2.8.5", "express": "^4.17.1", "express-promise-router": "^4.1.0", - "fs-extra": "^9.0.0", + "fs-extra": "^10.0.0", "helmet": "^4.0.0", "lodash": "^4.17.15", "morgan": "^1.10.0", diff --git a/plugins/scaffolder-backend/api-report.md b/plugins/scaffolder-backend/api-report.md index 792913749b..7528388a47 100644 --- a/plugins/scaffolder-backend/api-report.md +++ b/plugins/scaffolder-backend/api-report.md @@ -143,6 +143,12 @@ export function createFetchPlainAction(options: { integrations: ScmIntegrations; }): TemplateAction; +// @public (undocumented) +export const createFilesystemDeleteAction: () => TemplateAction; + +// @public (undocumented) +export const createFilesystemRenameAction: () => TemplateAction; + // @public (undocumented) export function createLegacyActions(options: Options): TemplateAction[]; diff --git a/plugins/scaffolder-backend/package.json b/plugins/scaffolder-backend/package.json index 9520ae12da..ab7ab1f552 100644 --- a/plugins/scaffolder-backend/package.json +++ b/plugins/scaffolder-backend/package.json @@ -47,7 +47,7 @@ "cross-fetch": "^3.0.6", "express": "^4.17.1", "express-promise-router": "^4.1.0", - "fs-extra": "^9.0.0", + "fs-extra": "^10.0.0", "git-url-parse": "^11.4.4", "globby": "^11.0.0", "handlebars": "^4.7.6", diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/createBuiltinActions.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/createBuiltinActions.ts index 598d36938e..46b8db72b4 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/createBuiltinActions.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/createBuiltinActions.ts @@ -24,6 +24,10 @@ import { } from './catalog'; import { createDebugLogAction } from './debug'; import { createFetchCookiecutterAction, createFetchPlainAction } from './fetch'; +import { + createFilesystemDeleteAction, + createFilesystemRenameAction, +} from './filesystem'; import { createPublishAzureAction, createPublishBitbucketAction, @@ -68,5 +72,7 @@ export const createBuiltinActions = (options: { createDebugLogAction(), createCatalogRegisterAction({ catalogClient, integrations }), createCatalogWriteAction(), + createFilesystemDeleteAction(), + createFilesystemRenameAction(), ]; }; diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/filesystem/delete.test.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/filesystem/delete.test.ts new file mode 100644 index 0000000000..ba8a145721 --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/filesystem/delete.test.ts @@ -0,0 +1,127 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as os from 'os'; +import mockFs from 'mock-fs'; +import { resolve as resolvePath } from 'path'; +import { createFilesystemDeleteAction } from './delete'; +import { getVoidLogger } from '@backstage/backend-common'; +import { PassThrough } from 'stream'; +import fs from 'fs-extra'; + +const root = os.platform() === 'win32' ? 'C:\\rootDir' : '/rootDir'; +const workspacePath = resolvePath(root, 'my-workspace'); + +describe('fs:delete', () => { + const action = createFilesystemDeleteAction(); + + const mockContext = { + input: { + files: ['unit-test-a.js', 'unit-test-b.js'], + }, + workspacePath, + logger: getVoidLogger(), + logStream: new PassThrough(), + output: jest.fn(), + createTemporaryDirectory: jest.fn(), + }; + + beforeEach(() => { + jest.restoreAllMocks(); + + mockFs({ + [workspacePath]: { + 'unit-test-a.js': 'hello', + 'unit-test-b.js': 'world', + 'a-folder': { + 'unit-test-in-a-folder.js2': 'content', + }, + }, + }); + }); + + afterEach(() => { + mockFs.restore(); + }); + + it('should throw an error when files is not an array', async () => { + await expect( + action.handler({ + ...mockContext, + input: { files: undefined }, + }), + ).rejects.toThrow(/files must be an Array/); + + await expect( + action.handler({ + ...mockContext, + input: { files: {} }, + }), + ).rejects.toThrow(/files must be an Array/); + + await expect( + action.handler({ + ...mockContext, + input: { files: '' }, + }), + ).rejects.toThrow(/files must be an Array/); + + await expect( + action.handler({ + ...mockContext, + input: { files: null }, + }), + ).rejects.toThrow(/files must be an Array/); + }); + + it('should throw when file name is not relative to the workspace', async () => { + await expect( + action.handler({ + ...mockContext, + input: { files: ['/foo/../../../index.js'] }, + }), + ).rejects.toThrow( + /Relative path is not allowed to refer to a directory outside its parent/, + ); + + await expect( + action.handler({ + ...mockContext, + input: { files: ['../../../index.js'] }, + }), + ).rejects.toThrow( + /Relative path is not allowed to refer to a directory outside its parent/, + ); + }); + + it('should call fs.rm with the correct values', async () => { + const files = ['unit-test-a.js', 'unit-test-b.js']; + + files.forEach(file => { + const filePath = resolvePath(workspacePath, file); + const fileExists = fs.existsSync(filePath); + expect(fileExists).toBe(true); + }); + + await action.handler(mockContext); + + files.forEach(file => { + const filePath = resolvePath(workspacePath, file); + const fileExists = fs.existsSync(filePath); + expect(fileExists).toBe(false); + }); + }); +}); diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/filesystem/delete.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/filesystem/delete.ts new file mode 100644 index 0000000000..068f777ec5 --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/filesystem/delete.ts @@ -0,0 +1,59 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { createTemplateAction } from '../../createTemplateAction'; +import { InputError } from '@backstage/errors'; +import { resolveSafeChildPath } from '@backstage/backend-common'; +import fs from 'fs-extra'; + +export const createFilesystemDeleteAction = () => { + return createTemplateAction<{ files: string[] }>({ + id: 'fs:delete', + description: 'Deletes files and directories from the workspace', + schema: { + input: { + required: ['files'], + type: 'object', + properties: { + files: { + title: 'Files', + description: 'A list of files and directories that will be deleted', + type: 'array', + items: { + type: 'string', + }, + }, + }, + }, + }, + async handler(ctx) { + if (!Array.isArray(ctx.input?.files)) { + throw new InputError('files must be an Array'); + } + + for (const file of ctx.input.files) { + const filepath = resolveSafeChildPath(ctx.workspacePath, file); + + try { + await fs.remove(filepath); + ctx.logger.info(`File ${filepath} deleted successfully`); + } catch (err) { + ctx.logger.error(`Failed to delete file ${filepath}:`, err); + throw err; + } + } + }, + }); +}; diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/filesystem/index.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/filesystem/index.ts new file mode 100644 index 0000000000..84e477b16b --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/filesystem/index.ts @@ -0,0 +1,18 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { createFilesystemDeleteAction } from './delete'; +export { createFilesystemRenameAction } from './rename'; diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/filesystem/rename.test.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/filesystem/rename.test.ts new file mode 100644 index 0000000000..8452735083 --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/filesystem/rename.test.ts @@ -0,0 +1,216 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as os from 'os'; +import mockFs from 'mock-fs'; +import { resolve as resolvePath } from 'path'; +import { createFilesystemRenameAction } from './rename'; +import { getVoidLogger } from '@backstage/backend-common'; +import { PassThrough } from 'stream'; +import fs from 'fs-extra'; + +const root = os.platform() === 'win32' ? 'C:\\rootDir' : '/rootDir'; +const workspacePath = resolvePath(root, 'my-workspace'); + +describe('fs:rename', () => { + const action = createFilesystemRenameAction(); + + const mockInputFiles = [ + { + from: 'unit-test-a.js', + to: 'new-a.js', + }, + { + from: 'unit-test-b.js', + to: 'new-b.js', + }, + { + from: 'a-folder', + to: 'brand-new-folder', + }, + ]; + const mockContext = { + input: { + files: mockInputFiles, + }, + workspacePath, + logger: getVoidLogger(), + logStream: new PassThrough(), + output: jest.fn(), + createTemporaryDirectory: jest.fn(), + }; + + beforeEach(() => { + jest.restoreAllMocks(); + + mockFs({ + [workspacePath]: { + 'unit-test-a.js': 'hello', + 'unit-test-b.js': 'world', + 'unit-test-c.js': 'i will be overwritten :-(', + 'a-folder': { + 'file.md': 'content', + }, + }, + }); + }); + + afterEach(() => { + mockFs.restore(); + }); + + it('should throw an error when files is not an array', async () => { + await expect( + action.handler({ + ...mockContext, + input: { files: undefined }, + }), + ).rejects.toThrow(/files must be an Array/); + + await expect( + action.handler({ + ...mockContext, + input: { files: {} }, + }), + ).rejects.toThrow(/files must be an Array/); + + await expect( + action.handler({ + ...mockContext, + input: { files: '' }, + }), + ).rejects.toThrow(/files must be an Array/); + + await expect( + action.handler({ + ...mockContext, + input: { files: null }, + }), + ).rejects.toThrow(/files must be an Array/); + }); + + it('should throw an error when files have missing from/to', async () => { + await expect( + action.handler({ + ...mockContext, + input: { files: ['old.md'] }, + }), + ).rejects.toThrow(/each file must have a from and to property/); + + await expect( + action.handler({ + ...mockContext, + input: { files: [{ from: 'old.md' }] }, + }), + ).rejects.toThrow(/each file must have a from and to property/); + + await expect( + action.handler({ + ...mockContext, + input: { files: [{ to: 'new.md' }] }, + }), + ).rejects.toThrow(/each file must have a from and to property/); + }); + + it('should throw when file name is not relative to the workspace', async () => { + await expect( + action.handler({ + ...mockContext, + input: { files: [{ from: 'index.js', to: '/core/../../../index.js' }] }, + }), + ).rejects.toThrow( + /Relative path is not allowed to refer to a directory outside its parent/, + ); + + await expect( + action.handler({ + ...mockContext, + input: { files: [{ from: '/core/../../../index.js', to: 'index.js' }] }, + }), + ).rejects.toThrow( + /Relative path is not allowed to refer to a directory outside its parent/, + ); + }); + + it('should throw is trying to override by mistake', async () => { + const destFile = 'unit-test-c.js'; + const filePath = resolvePath(workspacePath, destFile); + const beforeContent = fs.readFileSync(filePath, 'utf-8'); + + await expect( + action.handler({ + ...mockContext, + input: { + files: [ + { + from: 'unit-test-a.js', + to: 'unit-test-c.js', + }, + ], + }, + }), + ).rejects.toThrow(/dest already exists/); + + const afterContent = fs.readFileSync(filePath, 'utf-8'); + + expect(beforeContent).toEqual(afterContent); + }); + + it('should call fs.move with the correct values', async () => { + mockInputFiles.forEach(file => { + const filePath = resolvePath(workspacePath, file.from); + const fileExists = fs.existsSync(filePath); + expect(fileExists).toBe(true); + }); + + await action.handler(mockContext); + + mockInputFiles.forEach(file => { + const filePath = resolvePath(workspacePath, file.from); + const fileExists = fs.existsSync(filePath); + expect(fileExists).toBe(false); + }); + }); + + it('should override when requested', async () => { + const sourceFile = 'unit-test-a.js'; + const destFile = 'unit-test-c.js'; + const sourceFilePath = resolvePath(workspacePath, sourceFile); + const destFilePath = resolvePath(workspacePath, destFile); + + const sourceBeforeContent = fs.readFileSync(sourceFilePath, 'utf-8'); + const destBeforeContent = fs.readFileSync(destFilePath, 'utf-8'); + + expect(sourceBeforeContent).not.toEqual(destBeforeContent); + + await action.handler({ + ...mockContext, + input: { + files: [ + { + from: sourceFile, + to: destFile, + overwrite: true, + }, + ], + }, + }); + + const destAfterContent = fs.readFileSync(destFilePath, 'utf-8'); + + expect(sourceBeforeContent).toEqual(destAfterContent); + }); +}); diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/filesystem/rename.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/filesystem/rename.ts new file mode 100644 index 0000000000..aaadd5df57 --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/filesystem/rename.ts @@ -0,0 +1,98 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { createTemplateAction } from '../../createTemplateAction'; +import { resolveSafeChildPath } from '@backstage/backend-common'; + +import { InputError } from '@backstage/errors'; +import { JsonObject } from '@backstage/config'; +import fs from 'fs-extra'; + +interface FilesToRename extends JsonObject { + from: string; + to: string; +} + +export const createFilesystemRenameAction = () => { + return createTemplateAction<{ files: FilesToRename }>({ + id: 'fs:rename', + description: 'Renames files and directories within the workspace', + schema: { + input: { + required: ['files'], + type: 'object', + properties: { + files: { + title: 'Files', + description: + 'A list of file and directory names that will be renamed', + type: 'array', + items: { + type: 'object', + required: ['from', 'to'], + properties: { + from: { + type: 'string', + title: 'The source location of the file to be renamed', + }, + to: { + type: 'string', + title: 'The destination of the new file', + }, + overwrite: { + type: 'boolean', + title: + 'Overwrite existing file or directory, default is false', + }, + }, + }, + }, + }, + }, + }, + async handler(ctx) { + if (!Array.isArray(ctx.input?.files)) { + throw new InputError('files must be an Array'); + } + + for (const file of ctx.input.files) { + if (!file.from || !file.to) { + throw new InputError('each file must have a from and to property'); + } + + const sourceFilepath = resolveSafeChildPath( + ctx.workspacePath, + file.from, + ); + const destFilepath = resolveSafeChildPath(ctx.workspacePath, file.to); + + try { + await fs.move(sourceFilepath, destFilepath, { + overwrite: file.overwrite ?? false, + }); + ctx.logger.info( + `File ${sourceFilepath} renamed to ${destFilepath} successfully`, + ); + } catch (err) { + ctx.logger.error( + `Failed to rename file ${sourceFilepath} to ${destFilepath}:`, + err, + ); + throw err; + } + } + }, + }); +}; diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/index.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/index.ts index 047bcb421d..a5d5092327 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/index.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/index.ts @@ -18,4 +18,5 @@ export * from './catalog'; export { createBuiltinActions } from './createBuiltinActions'; export * from './debug'; export * from './fetch'; +export * from './filesystem'; export * from './publish'; diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/azure.test.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/azure.test.ts index 2f468cda25..d3cb4ccafc 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/azure.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/azure.test.ts @@ -162,6 +162,29 @@ describe('publish:azure', () => { expect(initRepoAndPush).toHaveBeenCalledWith({ dir: mockContext.workspacePath, remoteUrl: 'https://dev.azure.com/organization/project/_git/repo', + defaultBranch: 'master', + auth: { username: 'notempty', password: 'tokenlols' }, + logger: mockContext.logger, + }); + }); + + it('should call initRepoAndPush with the correct default branch', async () => { + mockGitClient.createRepository.mockImplementation(() => ({ + remoteUrl: 'https://dev.azure.com/organization/project/_git/repo', + })); + + await action.handler({ + ...mockContext, + input: { + ...mockContext.input, + defaultBranch: 'main', + }, + }); + + expect(initRepoAndPush).toHaveBeenCalledWith({ + dir: mockContext.workspacePath, + remoteUrl: 'https://dev.azure.com/organization/project/_git/repo', + defaultBranch: 'master', auth: { username: 'notempty', password: 'tokenlols' }, logger: mockContext.logger, }); diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/azure.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/azure.ts index 21b2537614..eb1b0522c6 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/azure.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/azure.ts @@ -30,6 +30,7 @@ export function createPublishAzureAction(options: { return createTemplateAction<{ repoUrl: string; description?: string; + defaultBranch?: string; sourcePath?: string; }>({ id: 'publish:azure', @@ -48,6 +49,11 @@ export function createPublishAzureAction(options: { title: 'Repository Description', type: 'string', }, + defaultBranch: { + title: 'Default Branch', + type: 'string', + description: `Sets the default branch on the repository. The default value is 'master'`, + }, sourcePath: { title: 'Path within the workspace that will be used as the repository root. If omitted, the entire workspace will be published as the repository.', @@ -70,9 +76,9 @@ export function createPublishAzureAction(options: { }, }, async handler(ctx) { - const { owner, repo, host, organization } = parseRepoUrl( - ctx.input.repoUrl, - ); + const { repoUrl, defaultBranch = 'master' } = ctx.input; + + const { owner, repo, host, organization } = parseRepoUrl(repoUrl); if (!organization) { throw new InputError( @@ -120,6 +126,7 @@ export function createPublishAzureAction(options: { await initRepoAndPush({ dir: getRepoSourceDirectory(ctx.workspacePath, ctx.input.sourcePath), remoteUrl, + defaultBranch, auth: { username: 'notempty', password: integrationConfig.config.token, diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/bitbucket.test.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/bitbucket.test.ts index 00ada3a4bc..16b84737a7 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/bitbucket.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/bitbucket.test.ts @@ -286,6 +286,49 @@ describe('publish:bitbucket', () => { expect(initRepoAndPush).toHaveBeenCalledWith({ dir: mockContext.workspacePath, remoteUrl: 'https://bitbucket.org/owner/cloneurl', + defaultBranch: 'master', + auth: { username: 'x-token-auth', password: 'tokenlols' }, + logger: mockContext.logger, + }); + }); + + it('should call initAndPush with the correct default branch', async () => { + server.use( + rest.post( + 'https://api.bitbucket.org/2.0/repositories/owner/repo', + (_, res, ctx) => + res( + ctx.status(200), + ctx.set('Content-Type', 'application/json'), + ctx.json({ + links: { + html: { + href: 'https://bitbucket.org/owner/repo', + }, + clone: [ + { + name: 'https', + href: 'https://bitbucket.org/owner/cloneurl', + }, + ], + }, + }), + ), + ), + ); + + await action.handler({ + ...mockContext, + input: { + ...mockContext.input, + defaultBranch: 'main', + }, + }); + + expect(initRepoAndPush).toHaveBeenCalledWith({ + dir: mockContext.workspacePath, + remoteUrl: 'https://bitbucket.org/owner/cloneurl', + defaultBranch: 'main', auth: { username: 'x-token-auth', password: 'tokenlols' }, logger: mockContext.logger, }); diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/bitbucket.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/bitbucket.ts index ce937108c4..295f4d1574 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/bitbucket.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/bitbucket.ts @@ -190,6 +190,7 @@ export function createPublishBitbucketAction(options: { return createTemplateAction<{ repoUrl: string; description: string; + defaultBranch?: string; repoVisibility: 'private' | 'public'; sourcePath?: string; enableLFS: boolean; @@ -215,6 +216,11 @@ export function createPublishBitbucketAction(options: { type: 'string', enum: ['private', 'public'], }, + defaultBranch: { + title: 'Default Branch', + type: 'string', + description: `Sets the default branch on the repository. The default value is 'master'`, + }, sourcePath: { title: 'Path within the workspace that will be used as the repository root. If omitted, the entire workspace will be published as the repository.', @@ -245,6 +251,7 @@ export function createPublishBitbucketAction(options: { const { repoUrl, description, + defaultBranch = 'master', repoVisibility = 'private', enableLFS = false, } = ctx.input; @@ -288,6 +295,7 @@ export function createPublishBitbucketAction(options: { ? integrationConfig.config.appPassword : integrationConfig.config.token ?? '', }, + defaultBranch, logger: ctx.logger, }); diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.test.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.test.ts index 716fa704e9..910fa27b46 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.test.ts @@ -175,6 +175,36 @@ describe('publish:github', () => { expect(initRepoAndPush).toHaveBeenCalledWith({ dir: mockContext.workspacePath, remoteUrl: 'https://github.com/clone/url.git', + defaultBranch: 'master', + auth: { username: 'x-access-token', password: 'tokenlols' }, + logger: mockContext.logger, + }); + }); + + it('should call initRepoAndPush with the correct defaultBranch main', async () => { + mockGithubClient.users.getByUsername.mockResolvedValue({ + data: { type: 'User' }, + }); + + mockGithubClient.repos.createForAuthenticatedUser.mockResolvedValue({ + data: { + clone_url: 'https://github.com/clone/url.git', + html_url: 'https://github.com/html/url', + }, + }); + + await action.handler({ + ...mockContext, + input: { + ...mockContext.input, + defaultBranch: 'main', + }, + }); + + expect(initRepoAndPush).toHaveBeenCalledWith({ + dir: mockContext.workspacePath, + remoteUrl: 'https://github.com/clone/url.git', + defaultBranch: 'main', auth: { username: 'x-access-token', password: 'tokenlols' }, logger: mockContext.logger, }); @@ -430,4 +460,34 @@ describe('publish:github', () => { 'https://github.com/html/url/blob/master', ); }); + + it('should use main as default branch', async () => { + mockGithubClient.users.getByUsername.mockResolvedValue({ + data: { type: 'User' }, + }); + + mockGithubClient.repos.createForAuthenticatedUser.mockResolvedValue({ + data: { + clone_url: 'https://github.com/clone/url.git', + html_url: 'https://github.com/html/url', + }, + }); + + await action.handler({ + ...mockContext, + input: { + ...mockContext.input, + defaultBranch: 'main', + }, + }); + + expect(mockContext.output).toHaveBeenCalledWith( + 'remoteUrl', + 'https://github.com/clone/url.git', + ); + expect(mockContext.output).toHaveBeenCalledWith( + 'repoContentsUrl', + 'https://github.com/html/url/blob/main', + ); + }); }); diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.ts index 48d59024eb..7dd790cb62 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.ts @@ -45,6 +45,7 @@ export function createPublishGithubAction(options: { repoUrl: string; description?: string; access?: string; + defaultBranch?: string; sourcePath?: string; repoVisibility: 'private' | 'internal' | 'public'; collaborators: Collaborator[]; @@ -77,6 +78,11 @@ export function createPublishGithubAction(options: { type: 'string', enum: ['private', 'public', 'internal'], }, + defaultBranch: { + title: 'Default Branch', + type: 'string', + description: `Sets the default branch on the repository. The default value is 'master'`, + }, sourcePath: { title: 'Path within the workspace that will be used as the repository root. If omitted, the entire workspace will be published as the repository.', @@ -131,6 +137,7 @@ export function createPublishGithubAction(options: { description, access, repoVisibility = 'private', + defaultBranch = 'master', collaborators, topics, } = ctx.input; @@ -239,11 +246,12 @@ export function createPublishGithubAction(options: { } const remoteUrl = newRepo.clone_url; - const repoContentsUrl = `${newRepo.html_url}/blob/master`; + const repoContentsUrl = `${newRepo.html_url}/blob/${defaultBranch}`; await initRepoAndPush({ dir: getRepoSourceDirectory(ctx.workspacePath, ctx.input.sourcePath), remoteUrl, + defaultBranch, auth: { username: 'x-access-token', password: token, @@ -257,10 +265,11 @@ export function createPublishGithubAction(options: { client, repoName: newRepo.name, logger: ctx.logger, + defaultBranch, }); } catch (e) { - throw new Error( - `Failed to add branch protection to '${newRepo.name}', ${e}`, + ctx.logger.warn( + `Skipping: default branch protection on '${newRepo.name}', ${e.message}`, ); } diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/gitab.test.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/gitlab.test.ts similarity index 88% rename from plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/gitab.test.ts rename to plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/gitlab.test.ts index 79de6fd669..67064efde7 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/gitab.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/gitlab.test.ts @@ -139,6 +139,30 @@ describe('publish:gitlab', () => { expect(initRepoAndPush).toHaveBeenCalledWith({ dir: mockContext.workspacePath, + defaultBranch: 'master', + remoteUrl: 'http://mockurl.git', + auth: { username: 'oauth2', password: 'tokenlols' }, + logger: mockContext.logger, + }); + }); + + it('should call initRepoAndPush with the correct default branch', async () => { + mockGitlabClient.Namespaces.show.mockResolvedValue({ id: 1234 }); + mockGitlabClient.Projects.create.mockResolvedValue({ + http_url_to_repo: 'http://mockurl.git', + }); + + await action.handler({ + ...mockContext, + input: { + ...mockContext.input, + defaultBranch: 'main', + }, + }); + + expect(initRepoAndPush).toHaveBeenCalledWith({ + dir: mockContext.workspacePath, + defaultBranch: 'main', remoteUrl: 'http://mockurl.git', auth: { username: 'oauth2', password: 'tokenlols' }, logger: mockContext.logger, diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/gitlab.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/gitlab.ts index 3f24b98e6c..57c7fdf752 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/gitlab.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/gitlab.ts @@ -28,6 +28,7 @@ export function createPublishGitlabAction(options: { return createTemplateAction<{ repoUrl: string; + defaultBranch?: string; repoVisibility: 'private' | 'internal' | 'public'; sourcePath?: string; }>({ @@ -48,6 +49,11 @@ export function createPublishGitlabAction(options: { type: 'string', enum: ['private', 'public', 'internal'], }, + defaultBranch: { + title: 'Default Branch', + type: 'string', + description: `Sets the default branch on the repository. The default value is 'master'`, + }, sourcePath: { title: 'Path within the workspace that will be used as the repository root. If omitted, the entire workspace will be published as the repository.', @@ -70,7 +76,11 @@ export function createPublishGitlabAction(options: { }, }, async handler(ctx) { - const { repoUrl, repoVisibility = 'private' } = ctx.input; + const { + repoUrl, + repoVisibility = 'private', + defaultBranch = 'master', + } = ctx.input; const { owner, repo, host } = parseRepoUrl(repoUrl); @@ -114,6 +124,7 @@ export function createPublishGitlabAction(options: { await initRepoAndPush({ dir: getRepoSourceDirectory(ctx.workspacePath, ctx.input.sourcePath), remoteUrl: http_url_to_repo as string, + defaultBranch, auth: { username: 'oauth2', password: integrationConfig.config.token, diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/publish/helpers.ts b/plugins/scaffolder-backend/src/scaffolder/stages/publish/helpers.ts index 6eebb4692a..dff0feb5f2 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/publish/helpers.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/publish/helpers.ts @@ -24,11 +24,13 @@ export async function initRepoAndPush({ remoteUrl, auth, logger, + defaultBranch = 'master', }: { dir: string; remoteUrl: string; auth: { username: string; password: string }; logger: Logger; + defaultBranch?: string; }): Promise { const git = Git.fromAuth({ username: auth.username, @@ -38,6 +40,7 @@ export async function initRepoAndPush({ await git.init({ dir, + defaultBranch, }); const paths = await globby(['./**', './**/.*', '!.git'], { @@ -74,6 +77,7 @@ type BranchProtectionOptions = { owner: string; repoName: string; logger: Logger; + defaultBranch?: string; }; export const enableBranchProtectionOnDefaultRepoBranch = async ({ @@ -81,6 +85,7 @@ export const enableBranchProtectionOnDefaultRepoBranch = async ({ client, owner, logger, + defaultBranch = 'master', }: BranchProtectionOptions): Promise => { const tryOnce = async () => { try { @@ -97,7 +102,7 @@ export const enableBranchProtectionOnDefaultRepoBranch = async ({ }, owner, repo: repoName, - branch: 'master', + branch: defaultBranch, required_status_checks: { strict: true, contexts: [] }, restrictions: null, enforce_admins: true, diff --git a/plugins/scaffolder/src/api.ts b/plugins/scaffolder/src/api.ts index 85df77f2f5..37d78d0fda 100644 --- a/plugins/scaffolder/src/api.ts +++ b/plugins/scaffolder/src/api.ts @@ -245,7 +245,10 @@ export class ScaffolderClient implements ScaffolderApi { */ async listActions(): Promise { const baseUrl = await this.discoveryApi.getBaseUrl('scaffolder'); - const response = await fetch(`${baseUrl}/v2/actions`); + const token = await this.identityApi.getIdToken(); + const response = await fetch(`${baseUrl}/v2/actions`, { + headers: token ? { Authorization: `Bearer ${token}` } : {}, + }); if (!response.ok) { throw ResponseError.fromResponse(response); diff --git a/plugins/shortcuts/src/Shortcuts.tsx b/plugins/shortcuts/src/Shortcuts.tsx index 775b0b3809..69ab6a48ac 100644 --- a/plugins/shortcuts/src/Shortcuts.tsx +++ b/plugins/shortcuts/src/Shortcuts.tsx @@ -16,24 +16,19 @@ import React, { useMemo } from 'react'; import { useObservable } from 'react-use'; -import { makeStyles } from '@material-ui/core'; import PlayListAddIcon from '@material-ui/icons/PlaylistAdd'; import { ShortcutItem } from './ShortcutItem'; import { AddShortcut } from './AddShortcut'; import { shortcutsApiRef } from './api'; -import { Progress, SidebarItem } from '@backstage/core-components'; +import { + Progress, + SidebarItem, + SidebarScrollWrapper, +} from '@backstage/core-components'; import { useApi } from '@backstage/core-plugin-api'; -const useStyles = makeStyles({ - root: { - flex: '1 1 auto', - overflowY: 'scroll', - }, -}); - export const Shortcuts = () => { - const classes = useStyles(); const shortcutApi = useApi(shortcutsApiRef); const shortcuts = useObservable( useMemo(() => shortcutApi.shortcut$(), [shortcutApi]), @@ -50,7 +45,7 @@ export const Shortcuts = () => { }; return ( -
+ { /> )) )} -
+ ); }; diff --git a/plugins/techdocs-backend/package.json b/plugins/techdocs-backend/package.json index 0fda61b1cf..0f0752a0c2 100644 --- a/plugins/techdocs-backend/package.json +++ b/plugins/techdocs-backend/package.json @@ -40,7 +40,7 @@ "dockerode": "^3.2.1", "express": "^4.17.1", "express-promise-router": "^4.1.0", - "fs-extra": "^9.0.1", + "fs-extra": "^10.0.0", "knex": "^0.95.1", "winston": "^3.2.1" }, diff --git a/yarn.lock b/yarn.lock index dceed09a91..27a6fcc250 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1689,6 +1689,11 @@ resolved "https://registry.npmjs.org/@date-io/core/-/core-2.10.7.tgz#0fe1fa0ef02c827919e23c2802a4b25589ac522d" integrity sha512-EG/1qDiQvd12RoNJ6H+sZcHVswC/3uMx/ySvfaJ24vB30rLjkgHggEXbgMbfgki7wMuiQ/zXI8QlmF1k3kWRGQ== +"@date-io/core@^2.10.11": + version "2.10.11" + resolved "https://registry.npmjs.org/@date-io/core/-/core-2.10.11.tgz#b1a3d57730f3eaaab54d5658be4a71727297e357" + integrity sha512-keXQnwH0LM8wyvu+j5Z2KGK56D+eItjy7DnwuWl/oV+DM2UEYl0z5WhdPMpfswSyt/kjuPOzcVF/7u/skMLaoA== + "@date-io/date-fns@^1.1.0": version "1.3.13" resolved "https://registry.npmjs.org/@date-io/date-fns/-/date-fns-1.3.13.tgz#7798844041640ab393f7e21a7769a65d672f4735" @@ -1696,12 +1701,12 @@ dependencies: "@date-io/core" "^1.3.13" -"@date-io/luxon@1.x": - version "1.3.13" - resolved "https://registry.npmjs.org/@date-io/luxon/-/luxon-1.3.13.tgz#68f0134bb38ef486b2ed6df01981f814c633e28a" - integrity sha512-9wUrJCNSMZJeYAiH+dbb45oGpnHeFP7TOH/Lt26If47gjFCkjvyINzWx+K5AGsnlP0Qosxc7hkF1yLi6ecutxw== +"@date-io/luxon@2.x": + version "2.10.11" + resolved "https://registry.npmjs.org/@date-io/luxon/-/luxon-2.10.11.tgz#d0981b9fdf5e5f17f8ce59265a3ac6c335565fac" + integrity sha512-SS6SIkp0Y9GFwpQycCTUAyW3OZTW05CWI1DJu10hUzcg8SmjJfhjs7hQY3TOeW+JT6VtXGTVGwbWPUBJsNkhZg== dependencies: - "@date-io/core" "^1.3.13" + "@date-io/core" "^2.10.11" "@emotion/cache@^10.0.27": version "10.0.29" @@ -5437,9 +5442,9 @@ integrity sha512-/+CRPXpBDpo2RK9C68N3b2cOvO0Cf5B9aPijHsoDQTHivnGSObdOF2BRQOYjojWTDy6nQvMjmqRXIxH55VjxxA== "@types/archiver@^5.1.0": - version "5.1.0" - resolved "https://registry.npmjs.org/@types/archiver/-/archiver-5.1.0.tgz#869f4ce4028e49cf9a0243cf914415f4cc3d1f3d" - integrity sha512-baFOhanb/hxmcOd1Uey2TfFg43kTSmM6py1Eo7Rjbv/ivcl7PXLhY0QgXGf50Hx/eskGCFqPfhs/7IZLb15C5g== + version "5.3.0" + resolved "https://registry.npmjs.org/@types/archiver/-/archiver-5.3.0.tgz#2b34ba56d4d7102d256b922c7e91e09eab79db6f" + integrity sha512-qJ79qsmq7O/k9FYwsF6O1xVA1PeLV+9Bh3TYkVCu3VzMR6vN9JQkgEOh/rrQ0R+F4Ta+R3thHGewxQtFglwVfg== dependencies: "@types/glob" "*" @@ -5454,9 +5459,9 @@ integrity sha512-iIgQNzCm0v7QMhhe4Jjn9uRh+I6GoPmt03CbEtwx3ao8/EfoQcmgtqH4vQ5Db/lxiIGaWDv6nwvunuh0RyX0+A== "@types/aws4@^1.5.1": - version "1.5.1" - resolved "https://registry.npmjs.org/@types/aws4/-/aws4-1.5.1.tgz#361fadab198a030ab398269183ae3fa86e958ed9" - integrity sha1-Nh+tqxmKAwqzmCaRg64/qG6Vjtk= + version "1.5.2" + resolved "https://registry.npmjs.org/@types/aws4/-/aws4-1.5.2.tgz#34e35b4405a619b9205be3e7678963bc7c8a47db" + integrity sha512-r8+XOv0BKw3Br0oU6w9cfu21PaQq/5ZeXvMOivNQYJLc9WcPCpEFDaQu72QNEQjYv5Otu48VhjjnbtwrSmk/rg== "@types/babel__core@^7.0.0", "@types/babel__core@^7.1.7": version "7.1.9" @@ -6866,9 +6871,9 @@ integrity sha512-/emrKCfQMQmFCqRqqBJ0JueHBT06jBRM3e8OgnvDUcvuExONujIk2hFA5dNsN9Nt41ljGVDdChvCydATZ+KOZw== "@types/yup@^0.29.8": - version "0.29.11" - resolved "https://registry.npmjs.org/@types/yup/-/yup-0.29.11.tgz#d654a112973f5e004bf8438122bd7e56a8e5cd7e" - integrity sha512-9cwk3c87qQKZrT251EDoibiYRILjCmxBvvcb4meofCmx1vdnNcR9gyildy5vOHASpOKMsn42CugxUvcwK5eu1g== + version "0.29.12" + resolved "https://registry.npmjs.org/@types/yup/-/yup-0.29.12.tgz#59c9577bea11d2b3d78717ea7591caacad6dfa1b" + integrity sha512-fA7bXyBzWEAgOwX2SD/5/iaZY/4In0EvJEzFmBWzaGNF4vxr8d5iOFUMFBpL4cMEmlSx2wW9ginJNnoZjE/vOg== "@types/zen-observable@^0.8.0", "@types/zen-observable@^0.8.2": version "0.8.2" @@ -13463,6 +13468,15 @@ fs-extra@^0.30.0: path-is-absolute "^1.0.0" rimraf "^2.2.8" +fs-extra@^10.0.0: + version "10.0.0" + resolved "https://registry.npmjs.org/fs-extra/-/fs-extra-10.0.0.tgz#9ff61b655dde53fb34a82df84bb214ce802e17c1" + integrity sha512-C5owb14u9eJwizKGdchcDUQeFtlSHHthBk8pbX9Vc1PFZrLombudjDnNns88aYslCyF6IY5SUw3Roz6xShcEIQ== + dependencies: + graceful-fs "^4.2.0" + jsonfile "^6.0.1" + universalify "^2.0.0" + fs-extra@^7.0.1, fs-extra@~7.0.1: version "7.0.1" resolved "https://registry.npmjs.org/fs-extra/-/fs-extra-7.0.1.tgz#4f189c44aa123b895f722804f55ea23eadc348e9" @@ -13481,7 +13495,7 @@ fs-extra@^8.1.0: jsonfile "^4.0.0" universalify "^0.1.0" -fs-extra@^9.0.0, fs-extra@^9.0.1, fs-extra@^9.1.0: +fs-extra@^9.0.0, fs-extra@^9.1.0: version "9.1.0" resolved "https://registry.npmjs.org/fs-extra/-/fs-extra-9.1.0.tgz#5954460c764a8da2094ba3554bf839e6b9a7c86d" integrity sha512-hcg3ZmepS30/7BSFqRvoo3DOMQu7IjqxO5nCDt+zM9XWjb33Wg7ziNT+Qvqbuc3+gWpzO02JubVyk2G4Zvo1OQ==