Merge branch 'master' into xcmetrics-dashboard-mvp

Signed-off-by: Niklas Granander <ngranander@spotify.com>

.changeset/angry-hounds-bow.md

@@ -0,0 +1,5 @@
+---
+'@backstage/integration': patch
+---
+
+Do not throw in `ScmIntegration` `byUrl` for invalid URLs

.changeset/beige-ladybugs-sip.md

@@ -0,0 +1,5 @@
+---
+'@backstage/core-components': patch
+---
+
+Improve UX of the Sidebar by adding SidebarScrollWrapper component allowing the user to scroll through Plugins & Shortcuts on smaller screens. Prevent the Sidebar from opening on click on small devices

.changeset/breezy-kings-applaud.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-pagerduty': patch
+---
+
+Update README

.changeset/brown-olives-hang.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-scaffolder-backend': patch
+---
+
+add default branch property for publish GitLab, Bitbucket and Azure actions

.changeset/eight-pots-remember.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-scaffolder-backend': patch
+---
+
+Added filesystem remove/rename built-in actions

.changeset/eighty-deers-jam.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-catalog-import': patch
+---
+
+More helpful error message when trying to import by folder from non-github

.changeset/fair-falcons-scream.md

@@ -0,0 +1,7 @@
+---
+'@backstage/core-components': patch
+'@backstage/plugin-explore': patch
+---
+
+- Enhanced core `Button` component to open external links in new tab.
+- Replaced the use of `Button` component from material by `core-components` in tools card.

.changeset/flat-donkeys-rhyme.md

@@ -0,0 +1,5 @@
+---
+'@backstage/catalog-model': minor
+---
+
+Changed the regex to validate names following the Kubernetes validation rule, this allow to be more permissive validating the name of the object in Backstage.

.changeset/fresh-elephants-tease.md

@@ -0,0 +1,68 @@
+---
+'@backstage/catalog-model': minor
+'@backstage/plugin-catalog-backend': minor
+'@backstage/plugin-scaffolder': minor
+'@backstage/plugin-scaffolder-backend': minor
+'@backstage/create-app': patch
+---
+
+Support for `Template` kinds with version `backstage.io/v1alpha1` has now been removed. This means that the old method of running templates with `Preparers`, `Templaters` and `Publishers` has also been removed. If you had any logic in these abstractions, they should now be moved to `actions` instead, and you can find out more about those in the [documentation](https://backstage.io/docs/features/software-templates/writing-custom-actions)
+
+If you need any help migrating existing templates, there's a [migration guide](https://backstage.io/docs/features/software-templates/migrating-from-v1alpha1-to-v1beta2). Reach out to us on Discord in the #support channel if you're having problems.
+
+The `scaffolder-backend` now no longer requires these `Preparers`, `Templaters`, and `Publishers` to be passed in, now all it needs is the `containerRunner`.
+
+Please update your `packages/backend/src/plugins/scaffolder.ts` like the following
+
+```diff
+- import {
+-  DockerContainerRunner,
+-  SingleHostDiscovery,
+- } from '@backstage/backend-common';
++ import { DockerContainerRunner } from '@backstage/backend-common';
+  import { CatalogClient } from '@backstage/catalog-client';
+- import {
+-   CookieCutter,
+-   CreateReactAppTemplater,
+-   createRouter,
+-   Preparers,
+-   Publishers,
+-   Templaters,
+- } from '@backstage/plugin-scaffolder-backend';
++ import { createRouter } from '@backstage/plugin-scaffolder-backend';
+  import Docker from 'dockerode';
+  import { Router } from 'express';
+  import type { PluginEnvironment } from '../types';
+
+  export default async function createPlugin({
+    config,
+    database,
+    reader,
++   discovery,
+  }: PluginEnvironment): Promise<Router> {
+    const dockerClient = new Docker();
+    const containerRunner = new DockerContainerRunner({ dockerClient });
+
+-   const cookiecutterTemplater = new CookieCutter({ containerRunner });
+-   const craTemplater = new CreateReactAppTemplater({ containerRunner });
+-   const templaters = new Templaters();
+
+-   templaters.register('cookiecutter', cookiecutterTemplater);
+-   templaters.register('cra', craTemplater);
+-
+-   const preparers = await Preparers.fromConfig(config, { logger });
+-   const publishers = await Publishers.fromConfig(config, { logger });
+
+-   const discovery = SingleHostDiscovery.fromConfig(config);
+    const catalogClient = new CatalogClient({ discoveryApi: discovery });
+
+    return await createRouter({
+-     preparers,
+-     templaters,
+-     publishers,
++     containerRunner,
+      logger,
+      config,
+      database,
+
+```

.changeset/fuzzy-bottles-greet.md

@@ -1,5 +0,0 @@
----
-'@backstage/core-app-api': patch
----
-
-Reintroduce export of `defaultConfigLoader`.

.changeset/gorgeous-timers-cover.md

@@ -0,0 +1,5 @@
+---
+'@backstage/techdocs-common': patch
+---
+
+Fix openStack swift publisher encoding issue. Remove utf8 forced encoding on binary files

.changeset/great-bears-work.md

@@ -0,0 +1,10 @@
+---
+'@backstage/plugin-scaffolder': patch
+---
+
+Added a `context` parameter to validator functions, letting them have access to
+the API holder.
+
+If you have implemented custom validators and use `createScaffolderFieldExtension`,
+your `validation` function can now optionally accept a third parameter,
+`context: { apiHolder: ApiHolder }`.

.changeset/happy-pugs-notice.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-catalog': patch
+---
+
+Change catalog page layout to use Grid components to improve responsiveness

.changeset/healthy-colts-watch.md

@@ -0,0 +1,7 @@
+---
+'@backstage/backend-test-utils': patch
+'@backstage/create-app': patch
+'@backstage/plugin-catalog-backend': patch
+---
+
+bump sqlite3 to 5.0.1

.changeset/honest-cars-glow.md

@@ -0,0 +1,54 @@
+---
+'@backstage/backend-common': patch
+---
+
+Added a `readUrl` method to the `UrlReader` interface that allows for complex response objects and is intended to replace the `read` method. This new method is currently optional to implement which allows for a soft migration to `readUrl` instead of `read` in the future.
+
+The main use case for `readUrl` returning an object instead of solely a read buffer is to allow for additional metadata such as ETag, which is a requirement for more efficient catalog processing.
+
+The `GithubUrlReader` and `GitlabUrlReader` readers fully implement `readUrl`. The other existing readers implement the new method but do not propagate or return ETags.
+
+While the `readUrl` method is not yet required, it will be in the future, and we already log deprecation warnings when custom `UrlReader` implementations that do not implement `readUrl` are used. We therefore recommend that any existing custom implementations are migrated to implement `readUrl`.
+
+The old `read` and the new `readUrl` methods can easily be implemented using one another, but we recommend moving the chunk of the implementation to the new `readUrl` method as `read` is being removed, for example this:
+
+```ts
+class CustomUrlReader implements UrlReader {
+  read(url: string): Promise<Buffer> {
+    const res = await fetch(url);
+
+    if (!res.ok) {
+      // error handling ...
+    }
+
+    return Buffer.from(await res.text());
+  }
+}
+```
+
+Can be migrated to something like this:
+
+```ts
+class CustomUrlReader implements UrlReader {
+  read(url: string): Promise<Buffer> {
+    const res = await this.readUrl(url);
+    return res.buffer();
+  }
+
+  async readUrl(
+    url: string,
+    _options?: ReadUrlOptions,
+  ): Promise<ReadUrlResponse> {
+    const res = await fetch(url);
+
+    if (!res.ok) {
+      // error handling ...
+    }
+
+    const buffer = Buffer.from(await res.text());
+    return { buffer: async () => buffer };
+  }
+}
+```
+
+While there is no usage of the ETag capability yet in the main Backstage packages, you can already add it to your custom implementations. To do so, refer to the documentation of the `readUrl` method and surrounding types, and the existing implementation in `packages/backend-common/src/reading/GithubUrlReader.ts`.

.changeset/large-mice-sin.md

@@ -0,0 +1,6 @@
+---
+'@backstage/plugin-catalog-backend-module-ldap': minor
+---
+
+Add extension points to the `LdapOrgReaderProcessor` to make it possible to do more advanced modifications
+of the ingested users and groups.

.changeset/late-pants-itch.md

@@ -0,0 +1,5 @@
+---
+'@backstage/core-components': patch
+---
+
+Show scroll bar of the sidebar wrapper only on hover

.changeset/long-olives-cheer.md

@@ -1,5 +0,0 @@
----
-'@backstage/plugin-auth-backend': patch
----
-
-Unbreak `.well-known` OIDC routes

.changeset/moody-garlics-whisper.md

@@ -0,0 +1,24 @@
+---
+'@backstage/plugin-scaffolder-backend': minor
+---
+
+Adding `config: Config` as a required argument to `createBuiltinActions` and downstream methods in order to support configuration of the default git author used for Scaffolder commits.
+
+The affected methods are:
+
+- `createBuiltinActions`
+- `createPublishGithubAction`
+- `createPublishGitlabAction`
+- `createPublishBitbucketAction`
+- `createPublishAzureAction`
+
+Call sites to these methods will need to be migrated to include the new `config` argument. See `createRouter` in `plugins/scaffolder-backend/src/service/router.ts` for an example of adding this new argument.
+
+To configure the default git author, use the `defaultAuthor` key under `scaffolder` in `app-config.yaml`:
+
+```yaml
+scaffolder:
+  defaultAuthor:
+    name: Example
+    email: example@example.com
+```

.changeset/nasty-crews-boil.md

@@ -1,5 +0,0 @@
----
-'@backstage/plugin-cost-insights': patch
----
-
-Replaced moment and dayjs with Luxon

.changeset/purple-carpets-fold.md

@@ -1,7 +0,0 @@
----
-'@backstage/techdocs-common': patch
-'@backstage/plugin-register-component': patch
-'@backstage/plugin-techdocs-backend': patch
----
-
-Update "service catalog" references to "software catalog"

.changeset/red-ladybugs-promise.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-scaffolder': patch
+---
+
+add support for uiSchema on dependent form fields

.changeset/rotten-cooks-try.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-ilert': patch
+---
+
+chore: bump `@date-io/luxon` from 1.3.13 to 2.10.11

.changeset/rotten-walls-cross.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-catalog-backend': patch
+---
+
+Make use of the new `readUrl` method on `UrlReader` from `@backstage/backend-common`.

.changeset/rude-pumpkins-hang.md

@@ -1,32 +0,0 @@
----
-'@backstage/core-components': patch
----
-
-Make `ErrorBoundary` display more helpful information about the error that
-occurred.
-
-The `slackChannel` (optional) prop can now be passed as an object on the form
-`{ name: string; href?: string; }` in addition to the old string form. If you
-are using the error boundary like
-
-```tsx
-<ErrorBoundary slackChannel="#support">
-  <InnerComponent>
-</ErrorBoundary>
-```
-
-you may like to migrate it to
-
-```tsx
-const support = {
-  name: '#support',
-  href: 'https://slack.com/channels/your-channel',
-};
-
-<ErrorBoundary slackChannel={support}>
-  <InnerComponent>
-</ErrorBoundary>
-```
-
-Also deprecated the prop `slackChannel` on `TabbedCard` and `InfoCard`, while
-adding the prop `errorBoundaryProps` to replace it.

.changeset/serious-falcons-greet.md

@@ -0,0 +1,5 @@
+---
+'@backstage/integration': patch
+---
+
+Fix downloads from repositories located at bitbucket.org

.changeset/shaggy-ties-carry.md

@@ -0,0 +1,5 @@
+---
+'@backstage/core-components': patch
+---
+
+Handle empty code blocks in markdown files so they don't fail rendering

.changeset/shy-garlics-tap.md

@@ -0,0 +1,5 @@
+---
+'@backstage/create-app': patch
+---
+
+Use SidebarScrollWrapper to improve responsiveness of the current sidebar. Change: Wrap a section of SidebarItems with this component to enable scroll for smaller screens. It can also be used in sidebar plugins (see shortcuts plugin for an example).

.changeset/silent-horses-fry.md

@@ -0,0 +1,5 @@
+---
+'@backstage/core-components': patch
+---
+
+Fix error in error panel, and console warnings about DOM nesting pre inside p

.changeset/six-birds-approve.md

@@ -0,0 +1,6 @@
+---
+'@backstage/core-components': patch
+'@backstage/plugin-org': patch
+---
+
+Add edit button to Group Profile Card

.changeset/sweet-terms-approve.md

@@ -2,4 +2,4 @@
 '@backstage/plugin-scaffolder-backend': patch
 ---
 
-Fix `catalog:write` on windows systems
+bump azure-devops-node to 10.2.2

.changeset/swift-fishes-taste.md

@@ -1,5 +0,0 @@
----
-'@backstage/plugin-catalog-backend': patch
----
-
-Ensure that emitted relations are deduplicated

.changeset/tasty-chairs-flash.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-scaffolder': patch
+---
+
+Pass through the `idToken` in `Authorization` Header for `listActions` request

.changeset/tasty-ghosts-raise.md

@@ -0,0 +1,8 @@
+---
+'@backstage/plugin-search-backend-node': patch
+---
+
+Change search scheduler from starting indexing in a fixed interval (for example
+every 60 seconds), to wait a fixed time between index runs.
+This makes sure that no second index process for the same document type is
+started when the previous one is still running.

.changeset/thirty-turtles-carry.md

@@ -1,5 +0,0 @@
----
-'@backstage/plugin-auth-backend': patch
----
-
-Show better error message when configs defined under auth.providers.<provider> are undefined.

.changeset/tricky-carrots-lick.md

@@ -1,24 +0,0 @@
----
-'@backstage/plugin-catalog-backend': minor
----
-
-Move `LdapOrgReaderProcessor` from `@backstage/plugin-catalog-backend`
-to `@backstage/plugin-catalog-backend-module-ldap`.
-
-The `LdapOrgReaderProcessor` isn't registered by default anymore, if
-you want to continue using it you have to register it manually at the catalog
-builder:
-
-1. Add dependency to `@backstage/plugin-catalog-backend-module-ldap` to the `package.json` of your backend.
-2. Add the processor to the catalog builder:
-
-```typescript
-// packages/backend/src/plugins/catalog.ts
-builder.addProcessor(
-  LdapOrgReaderProcessor.fromConfig(config, {
-    logger,
-  }),
-);
-```
-
-For more configuration details, see the [README of the `@backstage/plugin-catalog-backend-module-ldap` package](https://github.com/backstage/backstage/blob/master/plugins/catalog-backend-module-ldap/README.md).

.changeset/twelve-deers-explode.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-scaffolder-backend': patch
+---
+
+Export the `fetchContents` from scaffolder-backend

.changeset/two-pots-joke.md

@@ -1,5 +0,0 @@
----
-'@backstage/plugin-catalog': patch
----
-
-Get rid of flex console warning for IconLink

.changeset/violet-maps-smell.md

@@ -0,0 +1,6 @@
+---
+'@backstage/backend-common': patch
+'@backstage/plugin-scaffolder-backend': patch
+---
+
+add defaultBranch property for publish GitHub action

.changeset/wet-queens-deny.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-scaffolder-backend': patch
+---
+
+removing mandatory of protection for the default branch, that could be handled by the GitHub automation in async manner, thus throwing floating errors

.changeset/young-pigs-beg.md

@@ -0,0 +1,5 @@
+---
+'@backstage/core-components': patch
+---
+
+Add custom styles to scroll bar of the sidebar wrapper to fix flaky behaviour

.github/styles/vocab.txt

@@ -137,6 +137,7 @@ maintainership
 makefile
 md
 memcache
+memoized
 microservice
 microservices
 microsite
@@ -281,6 +282,7 @@ unregistration
 untracked
 upvote
 url
+URLs
 utils
 validator
 validators

ADOPTERS.md

@@ -1,7 +1,7 @@
 | Organization                                      | Contact                                                                                                                                                                                             | Description of Use                                                                                                                                                                                    |
 | ------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
 | [Spotify](https://www.spotify.com)                | [@leemills83](https://github.com/leemills83)                                                                                                                                                        | Main interface towards all of Spotify's infrastructure and technical documentation.                                                                                                                   |
-| [bol.com](https://www.bol.com)                    | [@RoyJacobs](https://github.com/RoyJacobs)                                                                                                                                                          | Initial work being done to unify platform tooling.                                                                                                                                                    |
+| [bol.com](https://www.bol.com)                    | [@sagacity](https://github.com/sagacity)                                                                                                                                                            | Initial work being done to unify platform tooling.                                                                                                                                                    |
 | [DFDS](https://www.dfds.com)                      | [@carlsendk](https://github.com/carlsendk)                                                                                                                                                          | V2 self-service platform.                                                                                                                                                                             |
 | [Roadie](https://roadie.io)                       | [@dtuite](https://github.com/dtuite)                                                                                                                                                                | Hosted, managed Backstage with easy set-up                                                                                                                                                            |
 | [Roku](https://www.roku.com)                      | [@timurista](https://github.com/timurista)                                                                                                                                                          | Initial work on Cloud engineering service platform.                                                                                                                                                   |
@@ -34,3 +34,6 @@
 | [FundApps](https://www.fundapps.co/)              | [Elliot Greenwood](https://github.com/egnwd)                                                                                                                                                        | Developer Portal - A place for us to keep track of our projects and documentation for all services and processes                                                                                      |
 | [DAZN](https://dazn.com/)                         | [Lou Bichard](https://twitter.com/loujaybee), [Marco Crivellaro](https://github.com/crivetechie), [Alex Hollerith](mailto:alex.hollerith@dazn.com)                                                  | Ingesting all of DAZN's repos for the catalog, migrating our internal platform apps (pull request boards, release information, inner source marketplace etc) to Backstage plugins (where applicable). |
 | [HelloFresh](https://www.hellofresh.de/)          | [@iammuho](https://github.com/iammuho), [@ElenaForester](https://github.com/ElenaForester), [@diegomarangoni](https://github.com/diegomarangoni)                                                    | Our developer portal at HelloFresh - Spread across an organisation of 500+ engineers globally.                                                                                                        |
+| [FactSet](https://www.factset.com/)               | [@kuangp](https://github.com/kuangp)                                                                                                                                                                | Developer portal to provide discoverability to all internal components, APIs, documentation, and scaffold templates with integrations to our internal infrastructure tools.                           |
+| [Workrise](https://www.workrise.com/)             | [Michael Rode](https://github.com/michaelrode)                                                                                                                                                      | Developer portal, main gateway to our infrastructure, documentation and internal tooling.                                                                                                             |
+| [RedVentures](https://www.redventures.com/)       | [Chris Diaz](https://github.com/codingdiaz)                                                                                                                                                         | Developer portal that brings everything an engineer needs to provide value into a single pane of glass.                                                                                               |

README.md

@@ -2,6 +2,8 @@
 
 # [Backstage](https://backstage.io)
 
+_During the month of July the majority of the maintainers will be on summer vacation 🏖️ Development will continue as usual, but expect a slower pace for discussions and PR reviews. Why not take this opportunity to [build a plugin](https://backstage.io/docs/plugins/)?_
+
 [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
 [![CNCF Status](https://img.shields.io/badge/cncf%20status-sandbox-blue.svg)](https://www.cncf.io/projects)
 [![Main CI Build](https://github.com/backstage/backstage/workflows/Main%20Master%20Build/badge.svg)](https://github.com/backstage/backstage/actions?query=workflow%3A%22Main+Master+Build%22)

app-config.yaml

@@ -253,6 +253,10 @@ catalog:
       target: ../catalog-model/examples/acme-corp.yaml
 
 scaffolder:
+  # Use to customize default commit author info used when new components are created
+  # defaultAuthor:
+  #   name: Scaffolder
+  #   email: scaffolder@backstage.io
   github:
     token: ${GITHUB_TOKEN}
     visibility: public # or 'internal' or 'private'

cypress/src/integration/catalog.ts

@@ -23,7 +23,7 @@ describe('Catalog', () => {
 
       cy.visit('/catalog');
 
-      cy.contains('Owned (8)').should('be.visible');
+      cy.contains('Owned (10)').should('be.visible');
     });
   });
 });

cypress/src/integration/integrations.ts

@@ -27,8 +27,10 @@ describe('Integrations', () => {
         type: 'url',
       });
 
+      cy.wait(5000);
+
       cy.visit('/catalog');
-      cy.contains('All').click();
+      cy.get('[data-testid="user-picker-all"]').click();
       cy.get('table').should('contain', 'github-repo');
       cy.get('table').should('contain', 'github-repo-nested');
     });
@@ -52,8 +54,10 @@ describe('Integrations', () => {
         type: 'url',
       });
 
+      cy.wait(5000);
+
       cy.visit('/catalog');
-      cy.contains('All').click();
+      cy.get('[data-testid="user-picker-all"]').click();
       cy.get('table').should('contain', 'gitlab-repo');
       cy.get('table').should('contain', 'gitlab-repo-nested');
     });
@@ -67,8 +71,10 @@ describe('Integrations', () => {
         type: 'url',
       });
 
+      cy.wait(5000);
+
       cy.visit('/catalog');
-      cy.contains('All').click();
+      cy.get('[data-testid="user-picker-all"]').click();
       cy.get('table').should('contain', 'bitbucket-repo');
       cy.get('table').should('contain', 'bitbucket-repo-nested');
     });

docs/deployment/docker.md

@@ -59,16 +59,17 @@ Once the host build is complete, we are ready to build our image. The following
 FROM node:14-buster-slim
 
 WORKDIR /app
-
 # Copy repo skeleton first, to avoid unnecessary docker cache invalidation.
 # The skeleton contains the package.json of each package in the monorepo,
 # and along with yarn.lock and the root package.json, that's enough to run yarn install.
-ADD yarn.lock package.json packages/backend/dist/skeleton.tar.gz ./
+COPY yarn.lock package.json packages/backend/dist/skeleton.tar.gz ./
+RUN tar xzf skeleton.tar.gz && rm skeleton.tar.gz
 
 RUN yarn install --frozen-lockfile --production --network-timeout 300000 && rm -rf "$(yarn cache dir)"
 
 # Then copy the rest of the backend bundle, along with any other files we might want.
-ADD packages/backend/dist/bundle.tar.gz app-config.yaml ./
+COPY packages/backend/dist/bundle.tar.gz app-config.yaml ./
+RUN tar xzf bundle.tar.gz && rm bundle.tar.gz
 
 CMD ["node", "packages/backend", "--config", "app-config.yaml"]
 ```

docs/features/techdocs/how-to-guides.md

@@ -78,11 +78,6 @@ the repository. The archive does not have any git history attached to it. Also
 it is a compressed file. Hence the file size is significantly smaller than how
 much data git clone has to transfer.
 
-Caveat: Currently TechDocs sites built using URL Reader will be cached for 30
-minutes which means they will not be re-built if new changes are made within 30
-minutes. This cache invalidation will be replaced by commit timestamp based
-implementation very soon.
-
 ## How to use a custom TechDocs home page?
 
 ### 1st way: TechDocsCustomHome with a custom configuration

docs/features/techdocs/troubleshooting.md

@@ -55,3 +55,20 @@ INFO    -  Start watching changes
 [I 210115 19:00:45 handlers:64] Start detecting changes
 INFO    -  Start detecting changes
 ```
+
+## PlantUML with `svg_object` doesn't render
+
+The [plantuml-markdown](https://pypi.org/project/plantuml-markdown/) MkDocs
+plugin available in
+[`mkdocs-techdocs-core`](https://github.com/backstage/mkdocs-techdocs-core)
+supports different formats for rendering diagrams. TechDocs does however not
+support all of them.
+
+The `svg_object` format renders a diagram as an HTML `<object>` tag but this is
+not allowed as it enables bad actors to inject malicious content into
+documentation pages. See
+[CVE-2021-32661](https://github.com/advisories/GHSA-gg96-f8wr-p89f) for more
+details.
+
+Instead use `svg_inline` which renders as an `<svg>` tag and provides the same
+benefits as `svg_object`.

docs/integrations/ldap/org.md

@@ -122,8 +122,8 @@ The DN under which users are stored, e.g.
 #### users.options
 
 The search options to use when sending the query to the server, when reading all
-users. All of the options are shown below, with their default values, but they
-are all optional.
+users. All the options are shown below, with their default values, but they are
+all optional.
 
 ```yaml
 options:
@@ -158,8 +158,8 @@ set:
 
 Mappings from well known entity fields, to LDAP attribute names. This is where
 you are able to define how to interpret the attributes of each LDAP result item,
-and to move them into the corresponding entity fields. All of the options are
-shown below, with their default values, but they are all optional.
+and to move them into the corresponding entity fields. All the options are shown
+below, with their default values, but they are all optional.
 
 If you leave out an optional mapping, it will still be copied using that default
 value. For example, even if you do not put in the field `displayName` in your
@@ -204,8 +204,8 @@ The DN under which groups are stored, e.g.
 #### groups.options
 
 The search options to use when sending the query to the server, when reading all
-groups. All of the options are shown below, with their default values, but they
-are all optional.
+groups. All the options are shown below, with their default values, but they are
+all optional.
 
 ```yaml
 options:
@@ -282,3 +282,35 @@ map:
   # the spec.children field of the entity.
   members: member
 ```
+
+## Customize the Processor
+
+In case you want to customize the ingested entities, the
+`LdapOrgReaderProcessor` allows to pass transformers for users and groups.
+
+1. Create a transformer:
+
+```ts
+export async function myGroupTransformer(
+  vendor: LdapVendor,
+  config: GroupConfig,
+  group: SearchEntry,
+): Promise<GroupEntity | undefined> {
+  // Transformations may change namespace, change entity naming pattern, fill
+  // profile with more or other details...
+
+  // Create the group entity on your own, or wrap the default transformer
+  return await defaultGroupTransformer(vendor, config, group);
+}
+```
+
+2. Configure the processor with the transformer:
+
+```ts
+builder.addProcessor(
+  LdapOrgReaderProcessor.fromConfig(config, {
+    logger,
+    groupTransformer: myGroupTransformer,
+  }),
+);
+```

docs/plugins/url-reader.md

@@ -0,0 +1,274 @@
+---
+id: url-reader
+title: URL Reader
+sidebar_label: URL Reader
+# prettier-ignore
+description: URL Reader is a backend core API responsible for reading files from external locations.
+---
+
+## Concept
+
+Some of the core plugins of Backstage have to read files from an external
+location. [Software Catalog](../features/software-catalog/index.md) has to read
+the [`catalog-info.yaml`](../features/software-catalog/descriptor-format.md)
+entity descriptor files to register and track an entity.
+[Software Templates](../features/software-templates/index.md) have to download
+the template skeleton files before creating a new component.
+[TechDocs](../features/techdocs/README.md) has to download the markdown source
+files before generating a documentation site.
+
+Since, the requirement for reading files is so essential for Backstage plugins,
+the
+[`@backstage/backend-common`](https://github.com/backstage/backstage/tree/master/packages/backend-common)
+package provides a dedicated API for reading from such URL based remote
+locations like GitHub, GitLab, Bitbucket, Google Cloud Storage, etc. This is
+commonly referred to as "URL Reader". It takes care of making authenticated
+requests to the remote host so that private files can be read securely. If users
+have [GitHub App based authentication](github-apps.md) set up, URL Reader even
+refreshes the token, to avoid reaching the GitHub API rate limit.
+
+As a result, plugin authors do not have to worry about any of these problems
+when trying to read files.
+
+## Interface
+
+When the Backstage backend starts, a new instance of URL Reader is created. You
+can see this in the index file of your Backstage backend
+i.e.`packages/backend/src/index.ts`.
+[Example](https://github.com/backstage/backstage/blob/ebbe91dbe79038a61d35cf6ed2d96e0e0d5a15f3/packages/backend/src/index.ts#L57)
+
+```ts
+// File: packages/backend/src/index.ts
+
+import { URLReaders } from '@backstage/backend-common';
+
+function makeCreateEnv(config: Config) {
+  // ....
+  const reader = UrlReaders.default({ logger, config });
+  //
+}
+```
+
+This instance contains all
+[the default URL Reader providers](https://github.com/backstage/backstage/blob/master/packages/backend-common/src/reading/UrlReaders.ts)
+in the backend-common package including GitHub, GitLab, Bitbucket, Azure, Google
+GCS. As the need arises, more URL Readers are being written to support different
+providers.
+
+The generic interface of a URL Reader instance looks like this.
+
+```ts
+export type UrlReader = {
+  /* Used to read a single file and return its content. */
+  read(url: string): Promise<Buffer>;
+  /**
+   * A replacement for the read method that supports options and complex responses.
+   *
+   * Use this whenever it is available, as the read method will be deprecated and
+   * eventually removed in the future.
+   */
+  readUrl?(url: string, options?: ReadUrlOptions): Promise<ReadUrlResponse>;
+
+  /* Used to read a file tree and download as a directory. */
+  readTree(url: string, options?: ReadTreeOptions): Promise<ReadTreeResponse>;
+  /* Used to search a file in a tree. */
+  search(url: string, options?: SearchOptions): Promise<SearchResponse>;
+};
+```
+
+## Using a URL Reader inside a plugin
+
+The `reader` instance is available in the backend plugin environment and passed
+on to all the backend plugins. You can see an
+[example](https://github.com/backstage/backstage/blob/b0be185369ebaad22255b7cdf18535d1d4ffd0e7/packages/backend/src/plugins/techdocs.ts#L31).
+When any of the methods on this instance is called with a URL, URL Reader
+extracts the host for that URL (e.g. `github.com`, `ghe.mycompany.com`, etc.).
+Using the
+[`@backstage/integration`](https://github.com/backstage/backstage/tree/master/packages/integration)
+package, it looks inside the
+[`integrations:`](https://github.com/backstage/backstage/blob/d5c83bb889b8142e343ebc4e4c0b90a02d1c1a3d/app-config.yaml#L134-L158)
+config of the `app-config.yaml` to find out how to work with the host based on
+the configs provided like authentication token, API base URL, etc.
+
+Make sure your plugin-specific backend file at
+`packages/backend/src/plugins/<PLUGIN>.ts` is forwarding the `reader` instance
+passed on as the `PluginEnvironment` to the actual plugin's `createRouter`
+function. See how this is done in
+[Catalog](https://github.com/backstage/backstage/blob/d5c83bb889b8142e343ebc4e4c0b90a02d1c1a3d/packages/backend/src/plugins/catalog.ts#L25-L27)
+and
+[TechDocs](https://github.com/backstage/backstage/blob/d5c83bb889b8142e343ebc4e4c0b90a02d1c1a3d/packages/backend/src/plugins/techdocs.ts#L31-L36)
+backend plugins.
+
+Once the reader instance is available inside the plugin, one of its methods can
+directly be used with a URL. Some example usages -
+
+- [`read`](https://github.com/backstage/backstage/blob/d5c83bb889b8142e343ebc4e4c0b90a02d1c1a3d/plugins/catalog-backend/src/ingestion/processors/codeowners/read.ts#L24-L33) -
+  Catalog using the `read` method to read the CODEOWNERS file in a repository.
+- [`readTree`](https://github.com/backstage/backstage/blob/d5c83bb889b8142e343ebc4e4c0b90a02d1c1a3d/packages/techdocs-common/src/helpers.ts#L198-L220) -
+  TechDocs using the `readTree` method to download markdown files in order to
+  generate the documentation site.
+- [`readTree`](https://github.com/backstage/backstage/blob/d5c83bb889b8142e343ebc4e4c0b90a02d1c1a3d/packages/techdocs-common/src/stages/prepare/url.ts#L33-L54) -
+  TechDocs using `NotModifiedError` to maintain cache and speed up and limit the
+  number of requests.
+- [`search`](https://github.com/backstage/backstage/blob/d5c83bb889b8142e343ebc4e4c0b90a02d1c1a3d/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.ts#L88-L108) -
+  Catalog using the `search` method to find files for a location URL containing
+  a glob pattern.
+
+## Writing a new URL Reader
+
+If the available URL Readers are not sufficient for your use case and you want
+to add a new URL Reader for any other provider, you are most welcome to
+contribute one!
+
+Feel free to use the
+[GitHub URL Reader](https://github.com/backstage/backstage/blob/d5c83bb889b8142e343ebc4e4c0b90a02d1c1a3d/packages/backend-common/src/reading/GithubUrlReader.ts)
+as a source of inspiration.
+
+### 1. Add an integration
+
+The provider for your new URL Reader can also be called an "integration" in
+Backstage. The `integrations:` section of your Backstage `app-config.yaml`
+config file is supposed to be the place where a Backstage integrator defines the
+host URL for the integration, authentication details and other integration
+related configurations.
+
+The `@backstage/integration` package is where most of the integration specific
+code lives, so that it is shareable across Backstage. Functions like "read the
+integrations config and process it", "construct headers for authenticated
+requests to the host" or "convert a plain file URL into its API URL for
+downloading the file" would live in this package.
+
+### 2. Create the URL Reader
+
+Create a new class which implements the
+[`UrlReader` type](https://github.com/backstage/backstage/blob/d5c83bb889b8142e343ebc4e4c0b90a02d1c1a3d/packages/backend-common/src/reading/types.ts#L21-L28)
+inside `@backstage/backend-common`. Create and export a static `factory` method
+which reads the integration config and returns a map of host URLs the new reader
+should be used for. See the
+[GitHub URL Reader](https://github.com/backstage/backstage/blob/d5c83bb889b8142e343ebc4e4c0b90a02d1c1a3d/packages/backend-common/src/reading/GithubUrlReader.ts#L50-L63)
+for example.
+
+### 3. Implement the methods
+
+We want to make sure all URL Readers behave in the same way. Hence if possible,
+all the methods of the `UrlReader` interface should be implemented. However it
+is okay to start by implementing just one of them and create issues for the
+remaining.
+
+#### read
+
+NOTE: Use `readUrl` instead of `read`.
+
+`read` method expects a user-friendly URL, something which can be copied from
+the browser naturally when a person is browsing the provider in their browser.
+
+- ✅ Valid URL :
+  `https://github.com/backstage/backstage/blob/master/ADOPTERS.md`
+- ❌ Not a valid URL :
+  `https://raw.githubusercontent.com/backstage/backstage/master/ADOPTERS.md`
+- ❌ Not a valid URL : `https://github.com/backstage/backstage/ADOPTERS.md`
+
+Upon receiving the URL, `read` converts the user-friendly URL into an API URL
+which can be used to request the provider's API.
+
+`read` then makes an authenticated request to the provider API and returns the
+file's content.
+
+#### readUrl
+
+`readUrl` is a new interface that allows complex response objects and is
+intended to replace the `read` method. This new method is currently optional to
+implement which allows for a soft migration to `readUrl` instead of `read` in
+the future.
+
+#### readTree
+
+`readTree` method also expects user-friendly URLs similar to `read` but the URL
+should point to a tree (could be the root of a repository or even a
+sub-directory).
+
+- ✅ Valid URL : `https://github.com/backstage/backstage`
+- ✅ Valid URL : `https://github.com/backstage/backstage/blob/master`
+- ✅ Valid URL : `https://github.com/backstage/backstage/blob/master/docs`
+
+Using the provider's API documentation, find out an API endpoint which can be
+used to download either a zip or a tarball. You can download the entire tree
+(e.g. a repository) and filter out in case the user is expecting only a
+sub-tree. But some APIs are smart enough to accept a path and return only a
+sub-tree in the downloaded archive.
+
+#### search
+
+`search` method expects a glob pattern of a URL and returns a list of files
+matching the query.
+
+- ✅ Valid URL :
+  `https://github.com/backstage/backstage/blob/master/**/catalog-info.yaml`
+- ✅ Valid URL : `https://github.com/backstage/backstage/blob/master/**/*.md`
+- ✅ Valid URL :
+  `https://github.com/backstage/backstage/blob/master/*/package.json`
+- ✅ Valid URL : `https://github.com/backstage/backstage/blob/master/READM`
+
+The core logic of `readTree` can be used here to extract all the files inside
+the tree and return the files matching the pattern in the `url`.
+
+### 4. Add to available URL Readers
+
+There are two ways to make your new URL Reader available for use.
+
+You can choose to make it open source, by updating the
+[`default` factory](https://github.com/backstage/backstage/blob/d5c83bb889b8142e343ebc4e4c0b90a02d1c1a3d/packages/backend-common/src/reading/UrlReaders.ts#L62-L81)
+method of URL Readers.
+
+But for something internal which you don't want to make open source, you can
+update your `packages/backend/src/index.ts` file and update how the `reader`
+instance is created.
+
+```ts
+// File: packages/backend/src/index.ts
+const reader = UrlReaders.default({
+  logger: root,
+  config,
+  // This is where your internal URL Readers would go.
+  factories: [myCustomReader.factory],
+});
+```
+
+### 5. Caching
+
+All of the methods above support an ETag based caching. If the method is called
+without an `etag`, the response contains an ETag of the resource (should ideally
+forward the ETag returned by the provider). If the method is called with an
+`etag`, it first compares the ETag and returns a `NotModifiedError` in case the
+resource has not been modified. This approach is very similar to the actual
+[ETag](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/ETag) and
+[If-None-Match](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/If-None-Match)
+HTTP headers.
+
+### 6. Debugging
+
+When debugging one of the URL Readers, you can straightforward use the
+[`reader` instance created](https://github.com/backstage/backstage/blob/ebbe91dbe79038a61d35cf6ed2d96e0e0d5a15f3/packages/backend/src/index.ts#L57)
+when the backend starts and call one of the methods with your debugging URL.
+
+```ts
+// File: packages/backend/src/index.ts
+
+async function main() {
+  // ...
+  const createEnv = makeCreateEnv(config);
+
+  const testReader = createEnv('test-url-reader').reader;
+  const response = await testReader.readUrl(
+    'https://github.com/backstage/backstage/blob/master/catalog-info.yaml',
+  );
+  console.log((await response.buffer()).toString());
+  // ...
+}
+```
+
+This will be run every time you restart the backend. Note that after any change
+in the URL Reader code, you need to kill the backend and restart, since the
+`reader` instance is memoized and does not update on hot module reloading. Also,
+there are a lot of unit tests written for the URL Readers, which you can make
+use of.

microsite/data/plugins/harbor.yaml

@@ -0,0 +1,13 @@
+---
+title: Harbor
+author: BESTSELLER
+authorUrl: bestsellerit.com
+category: Discovery
+description: This plugin will show you information about docker images within harbor.
+documentation: https://github.com/BESTSELLER/backstage-plugin-harbor/blob/master/README.md
+iconUrl: https://bestsellerit.com/img/terraform-harbor/goharbor.jpeg
+npmPackageName: '@bestsellerit/backstage-plugin-harbor'
+tags:
+  - goharbor
+  - harbor
+  - docker

microsite/data/plugins/scaffolder-backend-module-rails.yaml

@@ -0,0 +1,9 @@
+---
+title: Scaffolder Backend Module Rails
+author: Rogerio Angeliski
+authorUrl: https://angeliski.com.br/
+category: Scaffolder
+description: Here you can find all Rails related features to improve your scaffolder.
+documentation: https://github.com/backstage/backstage/blob/master/plugins/scaffolder-backend-module-rails/README.md
+iconUrl: img/rails-icon.png
+npmPackageName: '@backstage/plugin-scaffolder-backend-module-rails'

microsite/package.json

@@ -19,7 +19,7 @@
     "@spotify/prettier-config": "^10.0.0",
     "docusaurus": "^2.0.0-alpha.70",
     "js-yaml": "^4.1.0",
-    "prettier": "^2.3.1",
+    "prettier": "^2.3.2",
     "yarn-lock-check": "^1.0.5"
   },
   "prettier": "@spotify/prettier-config"

microsite/sidebars.json

@@ -166,7 +166,8 @@
           "plugins/proxying",
           "plugins/backend-plugin",
           "plugins/call-existing-api",
-          "plugins/github-apps"
+          "plugins/github-apps",
+          "plugins/url-reader"
         ]
       },
       {

microsite/static/css/custom.css

@@ -137,6 +137,20 @@ td {
   color: $navigatorItemTextColor;
 }
 
+/* search */
+[aria-expanded='true'] ~ .aa-without-1 {
+  display: block !important;
+}
+
+[aria-expanded='true'] ~ .aa-without-1 .aa-dataset-1:after {
+  content: 'No results';
+}
+
+.navSearchWrapper {
+  /* Prevents the search box from detaching from its icon when pushed down */
+  justify-content: flex-start;
+}
+
 /* header */
 .fixedHeaderContainer {
   position: fixed;
@@ -146,6 +160,12 @@ td {
   box-shadow: 3px 3px 8px rgba(38, 38, 38, 0.25);
   color: #fff;
 }
+
+.wrapper.headerWrapper {
+  /* Keep the header stretched across the screen on the mid breakpoint, since we need a bit more space */
+  max-width: 1400px;
+}
+
 @media only screen and (min-width: 1024px) {
   .fixedHeaderContainer {
     height: 66px;

microsite/yarn.lock

@@ -5207,10 +5207,10 @@ prepend-http@^2.0.0:
   resolved "https://registry.npmjs.org/prepend-http/-/prepend-http-2.0.0.tgz#e92434bfa5ea8c19f41cdfd401d741a3c819d897"
   integrity sha1-6SQ0v6XqjBn0HN/UAddBo8gZ2Jc=
 
-prettier@^2.3.1:
-  version "2.3.1"
-  resolved "https://registry.npmjs.org/prettier/-/prettier-2.3.1.tgz#76903c3f8c4449bc9ac597acefa24dc5ad4cbea6"
-  integrity sha512-p+vNbgpLjif/+D+DwAZAbndtRrR0md0MwfmOVN9N+2RgyACMT+7tfaRnT+WDPkqnuVwleyuBIG2XBxKDme3hPA==
+prettier@^2.3.2:
+  version "2.3.2"
+  resolved "https://registry.npmjs.org/prettier/-/prettier-2.3.2.tgz#ef280a05ec253712e486233db5c6f23441e7342d"
+  integrity sha512-lnJzDfJ66zkMy58OL5/NY5zp70S7Nz6KqcKkXYzn2tMVrNxvbqaBpg7H3qHaLxCJ5lNMsGuM8+ohS7cZrthdLQ==
 
 prismjs@^1.22.0:
   version "1.23.0"

mkdocs.yml

@@ -112,6 +112,7 @@ nav:
           - Backend plugin: 'plugins/backend-plugin.md'
           - Call existing API: 'plugins/call-existing-api.md'
           - GitHub Apps for Backend Authentication: 'plugins/github-apps.md'
+          - URL Reader: 'plugins/url-reader.md'
       - Testing:
           - Testing with Jest: 'plugins/testing.md'
       - Publishing:

package.json

@@ -61,7 +61,7 @@
     "command-exists": "^1.2.9",
     "concurrently": "^6.0.0",
     "eslint-plugin-notice": "^0.9.10",
-    "fs-extra": "^9.0.0",
+    "fs-extra": "^10.0.0",
     "husky": "^6.0.0",
     "lerna": "^4.0.0",
     "lint-staged": "^10.1.0",

packages/app/package.json

@@ -44,10 +44,10 @@
     "@material-ui/core": "^4.11.0",
     "@material-ui/icons": "^4.9.1",
     "@octokit/rest": "^18.5.3",
-    "@roadiehq/backstage-plugin-buildkite": "^1.0.3",
-    "@roadiehq/backstage-plugin-github-insights": "^1.1.11",
-    "@roadiehq/backstage-plugin-github-pull-requests": "^1.0.5",
-    "@roadiehq/backstage-plugin-travis-ci": "^1.0.0",
+    "@roadiehq/backstage-plugin-buildkite": "^1.0.4",
+    "@roadiehq/backstage-plugin-github-insights": "^1.1.15",
+    "@roadiehq/backstage-plugin-github-pull-requests": "^1.0.8",
+    "@roadiehq/backstage-plugin-travis-ci": "^1.0.4",
     "history": "^5.0.0",
     "prop-types": "^15.7.2",
     "react": "^16.12.0",

packages/app/src/components/Root/Root.tsx

@@ -39,6 +39,7 @@ import {
   SidebarItem,
   SidebarDivider,
   SidebarSpace,
+  SidebarScrollWrapper,
 } from '@backstage/core-components';
 
 const useSidebarLogoStyles = makeStyles({
@@ -88,10 +89,12 @@ export const Root = ({ children }: PropsWithChildren<{}>) => (
       <SidebarItem icon={CreateComponentIcon} to="create" text="Create..." />
       {/* End global nav */}
       <SidebarDivider />
-      <SidebarItem icon={MapIcon} to="tech-radar" text="Tech Radar" />
-      <SidebarItem icon={RuleIcon} to="lighthouse" text="Lighthouse" />
-      <SidebarItem icon={MoneyIcon} to="cost-insights" text="Cost Insights" />
-      <SidebarItem icon={GraphiQLIcon} to="graphiql" text="GraphiQL" />
+      <SidebarScrollWrapper>
+        <SidebarItem icon={MapIcon} to="tech-radar" text="Tech Radar" />
+        <SidebarItem icon={RuleIcon} to="lighthouse" text="Lighthouse" />
+        <SidebarItem icon={MoneyIcon} to="cost-insights" text="Cost Insights" />
+        <SidebarItem icon={GraphiQLIcon} to="graphiql" text="GraphiQL" />
+      </SidebarScrollWrapper>
       <SidebarDivider />
       <Shortcuts />
       <SidebarSpace />

packages/backend-common/CHANGELOG.md

@@ -1,5 +1,15 @@
 # @backstage/backend-common
 
+## 0.8.4
+
+### Patch Changes
+
+- 88d742eb8: Download archives as compressed tar files for GitLab to fix the `readTree` bug in TODO Plugin.
+- ab5cc376f: Add new `isChildPath` and `resolveSafeChildPath` exports
+- Updated dependencies
+  - @backstage/cli-common@0.1.2
+  - @backstage/integration@0.5.7
+
 ## 0.8.3
 
 ### Patch Changes

packages/backend-common/api-report.md

@@ -16,9 +16,10 @@ import { GithubCredentialsProvider } from '@backstage/integration';
 import { GitHubIntegration } from '@backstage/integration';
 import { GitLabIntegration } from '@backstage/integration';
 import * as http from 'http';
+import { isChildPath } from '@backstage/cli-common';
 import { JsonValue } from '@backstage/config';
 import { Knex } from 'knex';
-import { Logger } from 'winston';
+import { Logger as Logger_2 } from 'winston';
 import { MergeResult } from 'isomorphic-git';
 import { PushResult } from 'isomorphic-git';
 import { Readable } from 'stream';
@@ -41,6 +42,8 @@ export class AzureUrlReader implements UrlReader {
     // (undocumented)
     readTree(url: string, options?: ReadTreeOptions): Promise<ReadTreeResponse>;
     // (undocumented)
+    readUrl(url: string, _options?: ReadUrlOptions): Promise<ReadUrlResponse>;
+    // (undocumented)
     search(url: string, options?: SearchOptions): Promise<SearchResponse>;
     // (undocumented)
     toString(): string;
@@ -58,6 +61,8 @@ export class BitbucketUrlReader implements UrlReader {
     // (undocumented)
     readTree(url: string, options?: ReadTreeOptions): Promise<ReadTreeResponse>;
     // (undocumented)
+    readUrl(url: string, _options?: ReadUrlOptions): Promise<ReadUrlResponse>;
+    // (undocumented)
     search(url: string, options?: SearchOptions): Promise<SearchResponse>;
     // (undocumented)
     toString(): string;
@@ -124,7 +129,7 @@ export function errorHandler(options?: ErrorHandlerOptions): ErrorRequestHandler
 // @public (undocumented)
 export type ErrorHandlerOptions = {
     showStackTraces?: boolean;
-    logger?: Logger;
+    logger?: Logger_2;
     logClientErrors?: boolean;
 };
 
@@ -180,11 +185,12 @@ export class Git {
     static fromAuth: ({ username, password, logger, }: {
         username?: string | undefined;
         password?: string | undefined;
-        logger?: Logger | undefined;
+        logger?: Logger_2 | undefined;
     }) => Git;
     // (undocumented)
-    init({ dir }: {
+    init({ dir, defaultBranch, }: {
         dir: string;
+        defaultBranch?: string;
     }): Promise<void>;
     // (undocumented)
     merge({ dir, theirs, ours, author, committer, }: {
@@ -230,6 +236,8 @@ export class GithubUrlReader implements UrlReader {
     // (undocumented)
     readTree(url: string, options?: ReadTreeOptions): Promise<ReadTreeResponse>;
     // (undocumented)
+    readUrl(url: string, options?: ReadUrlOptions): Promise<ReadUrlResponse>;
+    // (undocumented)
     search(url: string, options?: SearchOptions): Promise<SearchResponse>;
     // (undocumented)
     toString(): string;
@@ -247,11 +255,15 @@ export class GitlabUrlReader implements UrlReader {
     // (undocumented)
     readTree(url: string, options?: ReadTreeOptions): Promise<ReadTreeResponse>;
     // (undocumented)
+    readUrl(url: string, options?: ReadUrlOptions): Promise<ReadUrlResponse>;
+    // (undocumented)
     search(url: string, options?: SearchOptions): Promise<SearchResponse>;
     // (undocumented)
     toString(): string;
 }
 
+export { isChildPath }
+
 // @public
 export function loadBackendConfig(options: Options): Promise<Config>;
 
@@ -274,7 +286,7 @@ export type PluginEndpointDiscovery = {
     getExternalBaseUrl(pluginId: string): Promise<string>;
 };
 
-// @public (undocumented)
+// @public
 export type ReadTreeResponse = {
     files(): Promise<ReadTreeResponseFile[]>;
     archive(): Promise<NodeJS.ReadableStream>;
@@ -289,11 +301,14 @@ export type ReadTreeResponseFile = {
 };
 
 // @public
-export function requestLoggingHandler(logger?: Logger): RequestHandler;
+export function requestLoggingHandler(logger?: Logger_2): RequestHandler;
 
 // @public
 export function resolvePackagePath(name: string, ...paths: string[]): string;
 
+// @public
+export function resolveSafeChildPath(base: string, path: string): string;
+
 // @public (undocumented)
 export type RunContainerOptions = {
     imageName: string;
@@ -322,7 +337,7 @@ export type ServiceBuilder = {
     loadConfig(config: ConfigReader): ServiceBuilder;
     setPort(port: number): ServiceBuilder;
     setHost(host: string): ServiceBuilder;
-    setLogger(logger: Logger): ServiceBuilder;
+    setLogger(logger: Logger_2): ServiceBuilder;
     enableCors(options: cors.CorsOptions): ServiceBuilder;
     setHttpsSettings(settings: HttpsSettings): ServiceBuilder;
     addRouter(root: string, router: Router | RequestHandler): ServiceBuilder;
@@ -360,6 +375,7 @@ export interface StatusCheckHandlerOptions {
 // @public
 export type UrlReader = {
     read(url: string): Promise<Buffer>;
+    readUrl?(url: string, options?: ReadUrlOptions): Promise<ReadUrlResponse>;
     readTree(url: string, options?: ReadTreeOptions): Promise<ReadTreeResponse>;
     search(url: string, options?: SearchOptions): Promise<SearchResponse>;
 };

packages/backend-common/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@backstage/backend-common",
   "description": "Common functionality library for Backstage backends",
-  "version": "0.8.3",
+  "version": "0.8.4",
   "main": "src/index.ts",
   "types": "src/index.ts",
   "private": false,
@@ -29,11 +29,11 @@
     "clean": "backstage-cli clean"
   },
   "dependencies": {
-    "@backstage/cli-common": "^0.1.1",
+    "@backstage/cli-common": "^0.1.2",
     "@backstage/config": "^0.1.5",
     "@backstage/config-loader": "^0.6.4",
     "@backstage/errors": "^0.1.1",
-    "@backstage/integration": "^0.5.6",
+    "@backstage/integration": "^0.5.7",
     "@google-cloud/storage": "^5.8.0",
     "@octokit/rest": "^18.5.3",
     "@types/cors": "^2.8.6",
@@ -47,7 +47,7 @@
     "dockerode": "^3.2.1",
     "express": "^4.17.1",
     "express-promise-router": "^4.1.0",
-    "fs-extra": "^9.0.1",
+    "fs-extra": "^10.0.0",
     "git-url-parse": "^11.4.4",
     "helmet": "^4.0.0",
     "isomorphic-git": "^1.8.0",
@@ -76,7 +76,7 @@
     }
   },
   "devDependencies": {
-    "@backstage/cli": "^0.7.1",
+    "@backstage/cli": "^0.7.3",
     "@backstage/test-utils": "^0.1.12",
     "@types/archiver": "^5.1.0",
     "@types/compression": "^1.7.0",

packages/backend-common/src/paths.ts

@@ -14,6 +14,8 @@
  * limitations under the License.
  */
 
+import { isChildPath } from '@backstage/cli-common';
+import { NotAllowedError } from '@backstage/errors';
 import { resolve as resolvePath } from 'path';
 
 /**
@@ -32,3 +34,27 @@ export function resolvePackagePath(name: string, ...paths: string[]) {
 
   return resolvePath(req.resolve(`${name}/package.json`), '..', ...paths);
 }
+
+/**
+ * Resolves a target path from a base path while guaranteeing that the result is
+ * a path that point to or within the base path. This is useful for resolving
+ * paths from user input, as it otherwise opens up for vulnerabilities.
+ *
+ * @param base The base directory to resolve the path from.
+ * @param path The target path, relative or absolute
+ * @returns A path that is guaranteed to point to or within the base path.
+ */
+export function resolveSafeChildPath(base: string, path: string): string {
+  const targetPath = resolvePath(base, path);
+
+  if (!isChildPath(base, targetPath)) {
+    throw new NotAllowedError(
+      'Relative path is not allowed to refer to a directory outside its parent',
+    );
+  }
+
+  return targetPath;
+}
+
+// Re-export isChildPath so that backend packages don't need to depend on cli-common
+export { isChildPath };

packages/backend-common/src/reading/AzureUrlReader.ts

@@ -36,6 +36,8 @@ import {
   SearchOptions,
   SearchResponse,
   UrlReader,
+  ReadUrlOptions,
+  ReadUrlResponse,
 } from './types';
 
 export class AzureUrlReader implements UrlReader {
@@ -78,6 +80,15 @@ export class AzureUrlReader implements UrlReader {
     throw new Error(message);
   }
 
+  async readUrl(
+    url: string,
+    _options?: ReadUrlOptions,
+  ): Promise<ReadUrlResponse> {
+    // TODO etag is not implemented yet.
+    const buffer = await this.read(url);
+    return { buffer: async () => buffer };
+  }
+
   async readTree(
     url: string,
     options?: ReadTreeOptions,

packages/backend-common/src/reading/BitbucketUrlReader.test.ts

@@ -74,7 +74,24 @@ describe('BitbucketUrlReader', () => {
   const worker = setupServer();
   msw.setupDefaultHandlers(worker);
 
-  describe('implementation', () => {
+  describe('readUrl', () => {
+    worker.use(
+      rest.get(
+        'https://api.bitbucket.org/2.0/repositories/backstage-verification/test-template/src/master/template.yaml',
+        (_, res, ctx) => res(ctx.status(200), ctx.body('foo')),
+      ),
+    );
+
+    it('should be able to readUrl', async () => {
+      const result = await bitbucketProcessor.readUrl(
+        'https://bitbucket.org/backstage-verification/test-template/src/master/template.yaml',
+      );
+      const buffer = await result.buffer();
+      expect(buffer.toString()).toBe('foo');
+    });
+  });
+
+  describe('read', () => {
     it('rejects unknown targets', async () => {
       await expect(
         bitbucketProcessor.read('https://not.bitbucket.com/apa'),
@@ -119,14 +136,14 @@ describe('BitbucketUrlReader', () => {
             ),
         ),
         rest.get(
-          'https://bitbucket.org/backstage/mock/get/master.tgz',
+          'https://bitbucket.org/backstage/mock/get/master.tar.gz',
           (_, res, ctx) =>
             res(
               ctx.status(200),
               ctx.set('Content-Type', 'application/zip'),
               ctx.set(
                 'content-disposition',
-                'attachment; filename=backstage-mock-12ab34cd56ef.tgz',
+                'attachment; filename=backstage-mock-12ab34cd56ef.tar.gz',
               ),
               ctx.body(repoBuffer),
             ),
@@ -142,7 +159,7 @@ describe('BitbucketUrlReader', () => {
             ),
         ),
         rest.get(
-          'https://api.bitbucket.mycompany.net/rest/api/1.0/projects/backstage/repos/mock/archive?format=tgz&prefix=mock&path=docs',
+          'https://api.bitbucket.mycompany.net/rest/api/1.0/projects/backstage/repos/mock/archive',
           (_, res, ctx) =>
             res(
               ctx.status(200),
@@ -304,14 +321,14 @@ describe('BitbucketUrlReader', () => {
             ),
         ),
         rest.get(
-          'https://bitbucket.org/backstage/mock/get/master.tgz',
+          'https://bitbucket.org/backstage/mock/get/master.tar.gz',
           (_, res, ctx) =>
             res(
               ctx.status(200),
               ctx.set('Content-Type', 'application/zip'),
               ctx.set(
                 'content-disposition',
-                'attachment; filename=backstage-mock-12ab34cd56ef.tgz',
+                'attachment; filename=backstage-mock-12ab34cd56ef.tar.gz',
               ),
               ctx.body(repoBuffer),
             ),
@@ -366,7 +383,7 @@ describe('BitbucketUrlReader', () => {
     beforeEach(() => {
       worker.use(
         rest.get(
-          'https://api.bitbucket.mycompany.net/rest/api/1.0/projects/backstage/repos/mock/archive?format=tgz&prefix=mock&path=docs',
+          'https://api.bitbucket.mycompany.net/rest/api/1.0/projects/backstage/repos/mock/archive',
           (_, res, ctx) =>
             res(
               ctx.status(200),

packages/backend-common/src/reading/BitbucketUrlReader.ts

@@ -36,6 +36,8 @@ import {
   SearchOptions,
   SearchResponse,
   UrlReader,
+  ReadUrlResponse,
+  ReadUrlOptions,
 } from './types';
 
 /**
@@ -99,6 +101,15 @@ export class BitbucketUrlReader implements UrlReader {
     throw new Error(message);
   }
 
+  async readUrl(
+    url: string,
+    _options?: ReadUrlOptions,
+  ): Promise<ReadUrlResponse> {
+    // TODO etag is not implemented yet.
+    const buffer = await this.read(url);
+    return { buffer: async () => buffer };
+  }
+
   async readTree(
     url: string,
     options?: ReadTreeOptions,

packages/backend-common/src/reading/FetchUrlReader.ts

@@ -19,6 +19,8 @@ import { NotFoundError } from '@backstage/errors';
 import {
   ReaderFactory,
   ReadTreeResponse,
+  ReadUrlOptions,
+  ReadUrlResponse,
   SearchResponse,
   UrlReader,
 } from './types';
@@ -73,6 +75,15 @@ export class FetchUrlReader implements UrlReader {
     throw new Error(message);
   }
 
+  async readUrl(
+    url: string,
+    _options?: ReadUrlOptions,
+  ): Promise<ReadUrlResponse> {
+    // TODO etag is not implemented yet.
+    const buffer = await this.read(url);
+    return { buffer: async () => buffer };
+  }
+
   async readTree(): Promise<ReadTreeResponse> {
     throw new Error('FetchUrlReader does not implement readTree');
   }

packages/backend-common/src/reading/GithubUrlReader.test.ts

@@ -118,7 +118,7 @@ describe('GithubUrlReader', () => {
 
       worker.use(
         rest.get(
-          'https://ghe.github.com/api/v3/repos/backstage/mock/tree/contents/?ref=main',
+          'https://ghe.github.com/api/v3/repos/backstage/mock/tree/contents/',
           (req, res, ctx) => {
             expect(req.headers.get('authorization')).toBe(
               mockHeaders.Authorization,
@@ -141,6 +141,113 @@ describe('GithubUrlReader', () => {
     });
   });
 
+  /*
+   * readUrl
+   */
+  describe('readUrl', () => {
+    it('should use the headers from the credentials provider to the fetch request when doing readUrl', async () => {
+      expect.assertions(2);
+
+      const mockHeaders = {
+        Authorization: 'bearer blah',
+        otherheader: 'something',
+      };
+
+      (mockCredentialsProvider.getCredentials as jest.Mock).mockResolvedValue({
+        headers: mockHeaders,
+      });
+
+      worker.use(
+        rest.get(
+          'https://ghe.github.com/api/v3/repos/backstage/mock/tree/contents/',
+          (req, res, ctx) => {
+            expect(req.headers.get('authorization')).toBe(
+              mockHeaders.Authorization,
+            );
+            expect(req.headers.get('otherheader')).toBe(
+              mockHeaders.otherheader,
+            );
+            return res(
+              ctx.status(200),
+              ctx.set('Content-Type', 'application/x-gzip'),
+              ctx.body('foo'),
+            );
+          },
+        ),
+      );
+
+      await gheProcessor.readUrl(
+        'https://github.com/backstage/mock/tree/blob/main',
+      );
+    });
+
+    it('should throw NotModified if GitHub responds with 304', async () => {
+      expect.assertions(4);
+
+      const mockHeaders = {
+        Authorization: 'bearer blah',
+        otherheader: 'something',
+      };
+
+      (mockCredentialsProvider.getCredentials as jest.Mock).mockResolvedValue({
+        headers: mockHeaders,
+      });
+
+      worker.use(
+        rest.get(
+          'https://ghe.github.com/api/v3/repos/backstage/mock/tree/contents/',
+          (req, res, ctx) => {
+            expect(req.headers.get('authorization')).toBe(
+              mockHeaders.Authorization,
+            );
+            expect(req.headers.get('otherheader')).toBe(
+              mockHeaders.otherheader,
+            );
+            expect(req.headers.get('if-none-match')).toBe('foo');
+            return res(
+              ctx.status(304),
+              ctx.set('Content-Type', 'application/x-gzip'),
+              ctx.body('foo'),
+            );
+          },
+        ),
+      );
+
+      await expect(
+        gheProcessor.readUrl(
+          'https://github.com/backstage/mock/tree/blob/main',
+          { etag: 'foo' },
+        ),
+      ).rejects.toThrow(NotModifiedError);
+    });
+
+    it('should return etag from the response', async () => {
+      (mockCredentialsProvider.getCredentials as jest.Mock).mockResolvedValue({
+        headers: {
+          Authorization: 'bearer blah',
+        },
+      });
+
+      worker.use(
+        rest.get(
+          'https://ghe.github.com/api/v3/repos/backstage/mock/tree/contents/',
+          (_req, res, ctx) => {
+            return res(
+              ctx.status(200),
+              ctx.set('Etag', 'foo'),
+              ctx.body('bar'),
+            );
+          },
+        ),
+      );
+
+      const response = await gheProcessor.readUrl(
+        'https://github.com/backstage/mock/tree/blob/main',
+      );
+      expect(response.etag).toBe('foo');
+    });
+  });
+
   /*
    * readTree
    */

packages/backend-common/src/reading/GithubUrlReader.ts

@@ -35,6 +35,8 @@ import {
   SearchResponse,
   SearchResponseFile,
   UrlReader,
+  ReadUrlOptions,
+  ReadUrlResponse,
 } from './types';
 
 export type GhRepoResponse = RestEndpointMethodTypes['repos']['get']['response']['data'];
@@ -77,6 +79,14 @@ export class GithubUrlReader implements UrlReader {
   }
 
   async read(url: string): Promise<Buffer> {
+    const response = await this.readUrl(url);
+    return response.buffer();
+  }
+
+  async readUrl(
+    url: string,
+    options?: ReadUrlOptions,
+  ): Promise<ReadUrlResponse> {
     const ghUrl = getGitHubFileFetchUrl(url, this.integration.config);
     const { headers } = await this.deps.credentialsProvider.getCredentials({
       url,
@@ -86,6 +96,7 @@ export class GithubUrlReader implements UrlReader {
       response = await fetch(ghUrl.toString(), {
         headers: {
           ...headers,
+          ...(options?.etag && { 'If-None-Match': options.etag }),
           Accept: 'application/vnd.github.v3.raw',
         },
       });
@@ -93,8 +104,15 @@ export class GithubUrlReader implements UrlReader {
       throw new Error(`Unable to read ${url}, ${e}`);
     }
 
+    if (response.status === 304) {
+      throw new NotModifiedError();
+    }
+
     if (response.ok) {
-      return Buffer.from(await response.text());
+      return {
+        buffer: async () => Buffer.from(await response.text()),
+        etag: response.headers.get('ETag') ?? undefined,
+      };
     }
 
     const message = `${url} could not be read as ${ghUrl}, ${response.status} ${response.statusText}`;

packages/backend-common/src/reading/GitlabUrlReader.test.ts

@@ -79,7 +79,7 @@ describe('GitlabUrlReader', () => {
   const worker = setupServer();
   msw.setupDefaultHandlers(worker);
 
-  describe('implementation', () => {
+  describe('read', () => {
     beforeEach(() => {
       worker.use(
         rest.get('*/api/v4/projects/:name', (_, res, ctx) =>
@@ -180,9 +180,56 @@ describe('GitlabUrlReader', () => {
     });
   });
 
+  describe('readUrl', () => {
+    const [{ reader }] = GitlabUrlReader.factory({
+      config: new ConfigReader({}),
+      logger,
+      treeResponseFactory,
+    });
+
+    it('should throw NotModified on HTTP 304', async () => {
+      worker.use(
+        rest.get('*/api/v4/projects/:name', (_, res, ctx) =>
+          res(ctx.status(200), ctx.json({ id: 12345 })),
+        ),
+        rest.get('*', (req, res, ctx) => {
+          expect(req.headers.get('If-None-Match')).toBe('999');
+          return res(ctx.status(304));
+        }),
+      );
+
+      await expect(
+        reader.readUrl!(
+          'https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/my/path/to/file.yaml',
+          {
+            etag: '999',
+          },
+        ),
+      ).rejects.toThrow(NotModifiedError);
+    });
+
+    it('should return etag in response', async () => {
+      worker.use(
+        rest.get('*/api/v4/projects/:name', (_, res, ctx) =>
+          res(ctx.status(200), ctx.json({ id: 12345 })),
+        ),
+        rest.get('*', (_req, res, ctx) => {
+          return res(ctx.status(200), ctx.set('ETag', '999'), ctx.body('foo'));
+        }),
+      );
+
+      const result = await reader.readUrl!(
+        'https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/my/path/to/file.yaml',
+      );
+      expect(result.etag).toBe('999');
+      const content = await result.buffer();
+      expect(content.toString()).toBe('foo');
+    });
+  });
+
   describe('readTree', () => {
     const archiveBuffer = fs.readFileSync(
-      path.resolve('src', 'reading', '__fixtures__', 'gitlab-archive.zip'),
+      path.resolve('src', 'reading', '__fixtures__', 'gitlab-archive.tar.gz'),
     );
 
     const projectGitlabApiResponse = {
@@ -205,7 +252,7 @@ describe('GitlabUrlReader', () => {
     beforeEach(() => {
       worker.use(
         rest.get(
-          'https://gitlab.com/api/v4/projects/backstage%2Fmock/repository/archive.zip?sha=main',
+          'https://gitlab.com/api/v4/projects/backstage%2Fmock/repository/archive',
           (_, res, ctx) =>
             res(
               ctx.status(200),
@@ -283,7 +330,7 @@ describe('GitlabUrlReader', () => {
           },
         ),
         rest.get(
-          'https://gitlab.mycompany.com/api/v4/projects/backstage%2Fmock/repository/archive.zip?sha=main',
+          'https://gitlab.mycompany.com/api/v4/projects/backstage%2Fmock/repository/archive',
           (_, res, ctx) =>
             res(
               ctx.status(200),
@@ -306,8 +353,8 @@ describe('GitlabUrlReader', () => {
       const files = await response.files();
       expect(files.length).toBe(2);
 
-      const mkDocsFile = await files[0].content();
-      const indexMarkdownFile = await files[1].content();
+      const indexMarkdownFile = await files[0].content();
+      const mkDocsFile = await files[1].content();
 
       expect(mkDocsFile.toString()).toBe('site_name: Test\n');
       expect(indexMarkdownFile.toString()).toBe('# Test\n');
@@ -318,7 +365,7 @@ describe('GitlabUrlReader', () => {
         'https://gitlab.com/backstage/mock',
       );
 
-      const dir = await response.dir({ targetDir: '/tmp' });
+      const dir = await response.dir({ targetDir: tmpDir });
 
       await expect(
         fs.readFile(path.join(dir, 'mkdocs.yml'), 'utf8'),
@@ -331,7 +378,7 @@ describe('GitlabUrlReader', () => {
     it('returns the wanted files from hosted gitlab', async () => {
       worker.use(
         rest.get(
-          'https://gitlab.mycompany.com/backstage/mock/-/archive/main.zip',
+          'https://gitlab.mycompany.com/backstage/mock/-/archive/main.tar.gz',
           (_, res, ctx) =>
             res(
               ctx.status(200),
@@ -375,7 +422,7 @@ describe('GitlabUrlReader', () => {
         'https://gitlab.com/backstage/mock/tree/main/docs',
       );
 
-      const dir = await response.dir({ targetDir: '/tmp' });
+      const dir = await response.dir({ targetDir: tmpDir });
 
       await expect(
         fs.readFile(path.join(dir, 'index.md'), 'utf8'),
@@ -454,7 +501,7 @@ describe('GitlabUrlReader', () => {
 
   describe('search', () => {
     const archiveBuffer = fs.readFileSync(
-      path.resolve('src', 'reading', '__fixtures__', 'gitlab-archive.zip'),
+      path.resolve('src', 'reading', '__fixtures__', 'gitlab-archive.tar.gz'),
     );
 
     const projectGitlabApiResponse = {
@@ -471,7 +518,7 @@ describe('GitlabUrlReader', () => {
     beforeEach(() => {
       worker.use(
         rest.get(
-          'https://gitlab.com/api/v4/projects/backstage%2Fmock/repository/archive.zip?sha=main',
+          'https://gitlab.com/api/v4/projects/backstage%2Fmock/repository/archive',
           (_, res, ctx) =>
             res(
               ctx.status(200),

packages/backend-common/src/reading/GitlabUrlReader.ts

@@ -34,6 +34,8 @@ import {
   SearchOptions,
   SearchResponse,
   UrlReader,
+  ReadUrlResponse,
+  ReadUrlOptions,
 } from './types';
 
 export class GitlabUrlReader implements UrlReader {
@@ -54,20 +56,37 @@ export class GitlabUrlReader implements UrlReader {
   ) {}
 
   async read(url: string): Promise<Buffer> {
+    const response = await this.readUrl(url);
+    return response.buffer();
+  }
+
+  async readUrl(
+    url: string,
+    options?: ReadUrlOptions,
+  ): Promise<ReadUrlResponse> {
     const builtUrl = await getGitLabFileFetchUrl(url, this.integration.config);
 
     let response: Response;
     try {
-      response = await fetch(
-        builtUrl,
-        getGitLabRequestOptions(this.integration.config),
-      );
+      response = await fetch(builtUrl, {
+        headers: {
+          ...getGitLabRequestOptions(this.integration.config).headers,
+          ...(options?.etag && { 'If-None-Match': options.etag }),
+        },
+      });
     } catch (e) {
       throw new Error(`Unable to read ${url}, ${e}`);
     }
 
+    if (response.status === 304) {
+      throw new NotModifiedError();
+    }
+
     if (response.ok) {
-      return Buffer.from(await response.text());
+      return {
+        buffer: async () => Buffer.from(await response.text()),
+        etag: response.headers.get('ETag') ?? undefined,
+      };
     }
 
     const message = `${url} could not be read as ${builtUrl}, ${response.status} ${response.statusText}`;
@@ -139,7 +158,7 @@ export class GitlabUrlReader implements UrlReader {
     const archiveGitLabResponse = await fetch(
       `${this.integration.config.apiBaseUrl}/projects/${encodeURIComponent(
         full_name,
-      )}/repository/archive.zip?sha=${branch}`,
+      )}/repository/archive?sha=${branch}`,
       getGitLabRequestOptions(this.integration.config),
     );
     if (!archiveGitLabResponse.ok) {
@@ -150,7 +169,7 @@ export class GitlabUrlReader implements UrlReader {
       throw new Error(message);
     }
 
-    return await this.deps.treeResponseFactory.fromZipArchive({
+    return await this.deps.treeResponseFactory.fromTarArchive({
       stream: (archiveGitLabResponse.body as unknown) as Readable,
       subpath: filepath,
       etag: commitSha,

packages/backend-common/src/reading/GoogleGcsUrlReader.ts

@@ -18,6 +18,8 @@ import { Storage } from '@google-cloud/storage';
 import {
   ReaderFactory,
   ReadTreeResponse,
+  ReadUrlOptions,
+  ReadUrlResponse,
   SearchResponse,
   UrlReader,
 } from './types';
@@ -90,6 +92,15 @@ export class GoogleGcsUrlReader implements UrlReader {
     }
   }
 
+  async readUrl(
+    url: string,
+    _options?: ReadUrlOptions,
+  ): Promise<ReadUrlResponse> {
+    // TODO etag is not implemented yet.
+    const buffer = await this.read(url);
+    return { buffer: async () => buffer };
+  }
+
   async readTree(): Promise<ReadTreeResponse> {
     throw new Error('GcsUrlReader does not implement readTree');
   }

packages/backend-common/src/reading/UrlReaderPredicateMux.ts

@@ -15,27 +15,35 @@
  */
 
 import { NotAllowedError } from '@backstage/errors';
+import { Logger } from 'winston';
 import {
   ReadTreeOptions,
   ReadTreeResponse,
+  ReadUrlOptions,
+  ReadUrlResponse,
   SearchOptions,
   SearchResponse,
   UrlReader,
   UrlReaderPredicateTuple,
 } from './types';
 
+const MIN_WARNING_INTERVAL_MS = 1000 * 60 * 15;
+
 /**
  * A UrlReader implementation that selects from a set of UrlReaders
  * based on a predicate tied to each reader.
  */
 export class UrlReaderPredicateMux implements UrlReader {
   private readonly readers: UrlReaderPredicateTuple[] = [];
+  private readonly readerWarnings: Map<UrlReader, number> = new Map();
+
+  constructor(private readonly logger: Logger) {}
 
   register(tuple: UrlReaderPredicateTuple): void {
     this.readers.push(tuple);
   }
 
-  read(url: string): Promise<Buffer> {
+  async read(url: string): Promise<Buffer> {
     const parsed = new URL(url);
 
     for (const { predicate, reader } of this.readers) {
@@ -47,6 +55,37 @@ export class UrlReaderPredicateMux implements UrlReader {
     throw new NotAllowedError(`Reading from '${url}' is not allowed`);
   }
 
+  async readUrl(
+    url: string,
+    options?: ReadUrlOptions,
+  ): Promise<ReadUrlResponse> {
+    const parsed = new URL(url);
+
+    for (const { predicate, reader } of this.readers) {
+      if (predicate(parsed)) {
+        if (reader.readUrl) {
+          return reader.readUrl(url, options);
+        }
+        const now = Date.now();
+        const lastWarned = this.readerWarnings.get(reader) ?? 0;
+        if (now > lastWarned + MIN_WARNING_INTERVAL_MS) {
+          this.readerWarnings.set(reader, now);
+          this.logger.warn(
+            `No implementation of readUrl found for ${reader}, this method will be required in the ` +
+              `future and will replace the 'read' method. See the changelog for more details here: ` +
+              'https://github.com/backstage/backstage/blob/master/packages/backend-common/CHANGELOG.md#085',
+          );
+        }
+        const buffer = await reader.read(url);
+        return {
+          buffer: async () => buffer,
+        };
+      }
+    }
+
+    throw new NotAllowedError(`Reading from '${url}' is not allowed`);
+  }
+
   async readTree(
     url: string,
     options?: ReadTreeOptions,

packages/backend-common/src/reading/UrlReaders.ts

@@ -43,7 +43,7 @@ export class UrlReaders {
    * Creates a UrlReader without any known types.
    */
   static create({ logger, config, factories }: CreateOptions): UrlReader {
-    const mux = new UrlReaderPredicateMux();
+    const mux = new UrlReaderPredicateMux(logger);
     const treeResponseFactory = DefaultReadTreeResponseFactory.create({
       config,
     });

packages/backend-common/src/reading/types.ts

@@ -22,8 +22,20 @@ import { Config } from '@backstage/config';
  * A generic interface for fetching plain data from URLs.
  */
 export type UrlReader = {
+  /* Used to read a single file and return its content. */
   read(url: string): Promise<Buffer>;
+
+  /**
+   * A replacement for the read method that supports options and complex responses.
+   *
+   * Use this whenever it is available, as the read method will be deprecated and
+   * eventually removed in the future.
+   */
+  readUrl?(url: string, options?: ReadUrlOptions): Promise<ReadUrlResponse>;
+
+  /* Used to read a file tree and download as a directory. */
   readTree(url: string, options?: ReadTreeOptions): Promise<ReadTreeResponse>;
+  /* Used to search a file in a tree using a glob pattern. */
   search(url: string, options?: SearchOptions): Promise<SearchResponse>;
 };
 
@@ -42,6 +54,40 @@ export type ReaderFactory = (options: {
   treeResponseFactory: ReadTreeResponseFactory;
 }) => UrlReaderPredicateTuple[];
 
+/**
+ * An options object for readUrl operations.
+ */
+export type ReadUrlOptions = {
+  /**
+   * An etag can be provided to check whether readUrl's response has changed from a previous execution.
+   *
+   * In the readUrl() response, an etag is returned along with the data. The etag is a unique identifer
+   * of the data, usually the commit SHA or etag from the target.
+   *
+   * When an etag is given in ReadUrlOptions, readUrl will first compare the etag against the etag
+   * on the target. If they match, readUrl will throw a NotModifiedError indicating that the readUrl
+   * response will not differ from the previous response which included this particular etag. If they
+   * do not match, readUrl will return the rest of ReadUrlResponse along with a new etag.
+   */
+  etag?: string;
+};
+
+/**
+ * A response object for readUrl operations.
+ */
+export type ReadUrlResponse = {
+  /**
+   * Returns the data that was read from the remote URL.
+   */
+  buffer(): Promise<Buffer>;
+
+  /**
+   * Etag returned by content provider.
+   * Can be used to compare and cache responses when doing subsequent calls.
+   */
+  etag?: string;
+};
+
 /**
  * An options object for readTree operations.
  */
@@ -66,14 +112,17 @@ export type ReadTreeOptions = {
    * In the readTree() response, an etag is returned along with the tree blob. The etag is a unique identifer
    * of the tree blob, usually the commit SHA or etag from the target.
    *
-   * When a etag is given in ReadTreeOptions, readTree will first compare the etag against the etag
+   * When an etag is given in ReadTreeOptions, readTree will first compare the etag against the etag
    * on the target branch. If they match, readTree will throw a NotModifiedError indicating that the readTree
-   * response will not differ from the previous response which included this particular etag. If they mismatch,
-   * readTree will return the rest of ReadTreeResponse along with a new etag.
+   * response will not differ from the previous response which included this particular etag. If they
+   * do not match, readTree will return the rest of ReadTreeResponse along with a new etag.
    */
   etag?: string;
 };
 
+/**
+ * A response object for readTree operations.
+ */
 export type ReadTreeResponse = {
   /**
    * files() returns an array of all the files inside the tree and corresponding functions to read their content.
@@ -87,7 +136,8 @@ export type ReadTreeResponse = {
   dir(options?: ReadTreeResponseDirOptions): Promise<string>;
 
   /**
-   * A unique identifer of the tree blob, usually the commit SHA or etag from the target.
+   * Etag returned by content provider.
+   * Can be used to compare and cache responses when doing subsequent calls.
    */
   etag: string;
 };

packages/backend-common/src/scm/git.test.ts

@@ -201,14 +201,16 @@ describe('Git', () => {
   describe('init', () => {
     it('should call isomorphic-git with the correct arguments', async () => {
       const dir = '/some/mock/dir';
+      const defaultBranch = 'master';
 
       const git = Git.fromAuth({});
 
-      await git.init({ dir });
+      await git.init({ dir, defaultBranch });
 
       expect(isomorphic.init).toHaveBeenCalledWith({
         fs,
         dir,
+        defaultBranch,
       });
     });
   });

packages/backend-common/src/scm/git.ts

@@ -150,12 +150,19 @@ export class Git {
     });
   }
 
-  async init({ dir }: { dir: string }): Promise<void> {
+  async init({
+    dir,
+    defaultBranch = 'master',
+  }: {
+    dir: string;
+    defaultBranch?: string;
+  }): Promise<void> {
     this.config.logger?.info(`Init git repository {dir=${dir}}`);
 
     return git.init({
       fs,
       dir,
+      defaultBranch,
     });
   }
 

packages/backend-test-utils/package.json

@@ -36,7 +36,7 @@
     "knex": "^0.95.1",
     "mysql2": "^2.2.5",
     "pg": "^8.3.0",
-    "sqlite3": "^5.0.0",
+    "sqlite3": "^5.0.1",
     "testcontainers": "^7.10.0",
     "uuid": "^8.0.0"
   },

packages/backend/CHANGELOG.md

@@ -1,5 +1,18 @@
 # example-backend
 
+## 0.2.35
+
+### Patch Changes
+
+- Updated dependencies
+  - @backstage/plugin-scaffolder-backend@0.12.4
+  - @backstage/backend-common@0.8.4
+  - @backstage/plugin-auth-backend@0.3.15
+  - @backstage/plugin-catalog-backend@0.11.0
+  - @backstage/plugin-techdocs-backend@0.8.5
+  - @backstage/catalog-client@0.3.15
+  - @backstage/plugin-kafka-backend@0.2.7
+
 ## 0.2.32
 
 ### Patch Changes

packages/backend/package.json

@@ -1,6 +1,6 @@
 {
   "name": "example-backend",
-  "version": "0.2.32",
+  "version": "0.2.35",
   "main": "dist/index.cjs.js",
   "types": "src/index.ts",
   "license": "Apache-2.0",
@@ -27,28 +27,30 @@
     "migrate:create": "knex migrate:make -x ts"
   },
   "dependencies": {
-    "@backstage/backend-common": "^0.8.2",
-    "@backstage/catalog-client": "^0.3.13",
+    "@backstage/backend-common": "^0.8.4",
+    "@backstage/catalog-client": "^0.3.15",
     "@backstage/catalog-model": "^0.8.2",
     "@backstage/config": "^0.1.5",
+    "@backstage/integration": "^0.5.6",
     "@backstage/plugin-app-backend": "^0.3.13",
-    "@backstage/plugin-auth-backend": "^0.3.12",
+    "@backstage/plugin-auth-backend": "^0.3.15",
     "@backstage/plugin-badges-backend": "^0.1.6",
-    "@backstage/plugin-catalog-backend": "^0.10.2",
+    "@backstage/plugin-catalog-backend": "^0.11.0",
     "@backstage/plugin-code-coverage-backend": "^0.1.6",
     "@backstage/plugin-graphql-backend": "^0.1.8",
     "@backstage/plugin-kubernetes-backend": "^0.3.8",
-    "@backstage/plugin-kafka-backend": "^0.2.6",
+    "@backstage/plugin-kafka-backend": "^0.2.7",
     "@backstage/plugin-proxy-backend": "^0.2.9",
     "@backstage/plugin-rollbar-backend": "^0.1.11",
-    "@backstage/plugin-scaffolder-backend": "^0.12.0",
+    "@backstage/plugin-scaffolder-backend": "^0.12.4",
+    "@backstage/plugin-scaffolder-backend-module-rails": "^0.1.1",
     "@backstage/plugin-search-backend": "^0.2.0",
     "@backstage/plugin-search-backend-node": "^0.2.0",
-    "@backstage/plugin-techdocs-backend": "^0.8.2",
+    "@backstage/plugin-techdocs-backend": "^0.8.5",
     "@backstage/plugin-todo-backend": "^0.1.6",
     "@gitbeaker/node": "^30.2.0",
     "@octokit/rest": "^18.5.3",
-    "azure-devops-node-api": "^10.1.1",
+    "azure-devops-node-api": "^10.2.2",
     "dockerode": "^3.2.1",
     "example-app": "^0.2.32",
     "express": "^4.17.1",
@@ -56,11 +58,11 @@
     "knex": "^0.95.1",
     "pg": "^8.3.0",
     "pg-connection-string": "^2.3.0",
-    "sqlite3": "^5.0.0",
+    "sqlite3": "^5.0.1",
     "winston": "^3.2.1"
   },
   "devDependencies": {
-    "@backstage/cli": "^0.7.0",
+    "@backstage/cli": "^0.7.3",
     "@types/dockerode": "^3.2.1",
     "@types/express": "^4.17.6",
     "@types/express-serve-static-core": "^4.17.5"

packages/backend/src/plugins/scaffolder.ts

@@ -14,19 +14,9 @@
  * limitations under the License.
  */
 
-import {
-  DockerContainerRunner,
-  SingleHostDiscovery,
-} from '@backstage/backend-common';
+import { DockerContainerRunner } from '@backstage/backend-common';
 import { CatalogClient } from '@backstage/catalog-client';
-import {
-  CookieCutter,
-  CreateReactAppTemplater,
-  createRouter,
-  Preparers,
-  Publishers,
-  Templaters,
-} from '@backstage/plugin-scaffolder-backend';
+import { createRouter } from '@backstage/plugin-scaffolder-backend';
 import Docker from 'dockerode';
 import { Router } from 'express';
 import type { PluginEnvironment } from '../types';
@@ -36,27 +26,15 @@ export default async function createPlugin({
   config,
   database,
   reader,
+  discovery,
 }: PluginEnvironment): Promise<Router> {
   const dockerClient = new Docker();
   const containerRunner = new DockerContainerRunner({ dockerClient });
 
-  const cookiecutterTemplater = new CookieCutter({ containerRunner });
-  const craTemplater = new CreateReactAppTemplater({ containerRunner });
-  const templaters = new Templaters();
-
-  templaters.register('cookiecutter', cookiecutterTemplater);
-  templaters.register('cra', craTemplater);
-
-  const preparers = await Preparers.fromConfig(config, { logger });
-  const publishers = await Publishers.fromConfig(config, { logger });
-
-  const discovery = SingleHostDiscovery.fromConfig(config);
   const catalogClient = new CatalogClient({ discoveryApi: discovery });
 
   return await createRouter({
-    preparers,
-    templaters,
-    publishers,
+    containerRunner,
     logger,
     config,
     database,

packages/catalog-client/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @backstage/catalog-client
 
+## 0.3.15
+
+### Patch Changes
+
+- ca080cab8: Don't crash if the entities response doesn't include the entities name and kind
+
 ## 0.3.14
 
 ### Patch Changes

packages/catalog-client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@backstage/catalog-client",
-  "version": "0.3.14",
+  "version": "0.3.15",
   "main": "src/index.ts",
   "types": "src/index.ts",
   "license": "Apache-2.0",
@@ -35,7 +35,7 @@
     "cross-fetch": "^3.0.6"
   },
   "devDependencies": {
-    "@backstage/cli": "^0.7.2",
+    "@backstage/cli": "^0.7.3",
     "@types/jest": "^26.0.7",
     "msw": "^0.29.0"
   },

packages/catalog-client/src/CatalogClient.test.ts

@@ -151,6 +151,26 @@ describe('CatalogClient', () => {
 
       expect(response.items).toEqual([]);
     });
+
+    it('handles field filtered entities', async () => {
+      server.use(
+        rest.get(`${mockBaseUrl}/entities`, (_req, res, ctx) => {
+          return res(ctx.json([{ apiVersion: '1' }, { apiVersion: '2' }]));
+        }),
+      );
+
+      const response = await client.getEntities(
+        {
+          fields: ['apiVersion'],
+        },
+        { token },
+      );
+
+      expect(response.items).toEqual([
+        { apiVersion: '1' },
+        { apiVersion: '2' },
+      ]);
+    });
   });
 
   describe('getLocationById', () => {

packages/catalog-client/src/CatalogClient.ts

@@ -92,6 +92,16 @@ export class CatalogClient implements CatalogApi {
     );
 
     const refCompare = (a: Entity, b: Entity) => {
+      // in case field filtering is used, these fields might not be part of the response
+      if (
+        a.metadata?.name === undefined ||
+        a.kind === undefined ||
+        b.metadata?.name === undefined ||
+        b.kind === undefined
+      ) {
+        return 0;
+      }
+
       const aRef = stringifyEntityRef(a);
       const bRef = stringifyEntityRef(b);
       if (aRef < bRef) {

packages/catalog-model/api-report.md

@@ -492,29 +492,6 @@ export { SystemEntityV1alpha1 }
 // @public (undocumented)
 export const systemEntityV1alpha1Validator: KindValidator;
 
-// @public (undocumented)
-interface TemplateEntityV1alpha1 extends Entity {
-    // (undocumented)
-    apiVersion: 'backstage.io/v1alpha1' | 'backstage.io/v1beta1';
-    // (undocumented)
-    kind: 'Template';
-    // (undocumented)
-    spec: {
-        type: string;
-        templater: string;
-        path?: string;
-        schema: JSONSchema;
-        owner?: string;
-    };
-}
-
-export { TemplateEntityV1alpha1 as TemplateEntity }
-
-export { TemplateEntityV1alpha1 }
-
-// @public (undocumented)
-export const templateEntityV1alpha1Validator: KindValidator;
-
 // @public (undocumented)
 export interface TemplateEntityV1beta2 extends Entity {
     // (undocumented)

packages/catalog-model/src/kinds/TemplateEntityV1alpha1.test.ts

@@ -1,106 +0,0 @@
-/*
- * Copyright 2020 The Backstage Authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import {
-  TemplateEntityV1alpha1,
-  templateEntityV1alpha1Validator as validator,
-} from './TemplateEntityV1alpha1';
-
-describe('templateEntityV1alpha1Validator', () => {
-  let entity: TemplateEntityV1alpha1;
-
-  beforeEach(() => {
-    entity = {
-      apiVersion: 'backstage.io/v1alpha1',
-      kind: 'Template',
-      metadata: {
-        name: 'test',
-      },
-      spec: {
-        type: 'website',
-        templater: 'cookiecutter',
-        schema: {
-          $schema: 'http://json-schema.org/draft-07/schema#',
-          required: ['storePath', 'owner'],
-          properties: {
-            owner: {
-              type: 'string',
-              title: 'Owner',
-              description: 'Who is going to own this component',
-            },
-            storePath: {
-              type: 'string',
-              title: 'Store path',
-              description: 'GitHub store path in org/repo format',
-            },
-          },
-        },
-        owner: 'team-a@example.com',
-      },
-    };
-  });
-
-  it('happy path: accepts valid data', async () => {
-    await expect(validator.check(entity)).resolves.toBe(true);
-  });
-
-  it('silently accepts v1beta1 as well', async () => {
-    (entity as any).apiVersion = 'backstage.io/v1beta1';
-    await expect(validator.check(entity)).resolves.toBe(true);
-  });
-
-  it('ignores unknown apiVersion', async () => {
-    (entity as any).apiVersion = 'backstage.io/v1beta0';
-    await expect(validator.check(entity)).resolves.toBe(false);
-  });
-
-  it('ignores unknown kind', async () => {
-    (entity as any).kind = 'Wizard';
-    await expect(validator.check(entity)).resolves.toBe(false);
-  });
-
-  it('rejects missing type', async () => {
-    delete (entity as any).spec.type;
-    await expect(validator.check(entity)).rejects.toThrow(/type/);
-  });
-
-  it('accepts any other type', async () => {
-    (entity as any).spec.type = 'hallo';
-    await expect(validator.check(entity)).resolves.toBe(true);
-  });
-
-  it('rejects empty type', async () => {
-    (entity as any).spec.type = '';
-    await expect(validator.check(entity)).rejects.toThrow(/type/);
-  });
-
-  it('rejects missing templater', async () => {
-    (entity as any).spec.templater = '';
-    await expect(validator.check(entity)).rejects.toThrow(/templater/);
-  });
-  it('accepts missing owner', async () => {
-    delete (entity as any).spec.owner;
-    await expect(validator.check(entity)).resolves.toBe(true);
-  });
-  it('rejects empty owner', async () => {
-    (entity as any).spec.owner = '';
-    await expect(validator.check(entity)).rejects.toThrow(/owner/);
-  });
-  it('rejects wrong type owner', async () => {
-    (entity as any).spec.owner = 5;
-    await expect(validator.check(entity)).rejects.toThrow(/owner/);
-  });
-});

packages/catalog-model/src/kinds/TemplateEntityV1alpha1.ts

@@ -1,36 +0,0 @@
-/*
- * Copyright 2020 The Backstage Authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import type { Entity } from '../entity/Entity';
-import schema from '../schema/kinds/Template.v1alpha1.schema.json';
-import type { JSONSchema } from '../types';
-import { ajvCompiledJsonSchemaValidator } from './util';
-
-export interface TemplateEntityV1alpha1 extends Entity {
-  apiVersion: 'backstage.io/v1alpha1' | 'backstage.io/v1beta1';
-  kind: 'Template';
-  spec: {
-    type: string;
-    templater: string;
-    path?: string;
-    schema: JSONSchema;
-    owner?: string;
-  };
-}
-
-export const templateEntityV1alpha1Validator = ajvCompiledJsonSchemaValidator(
-  schema,
-);

packages/catalog-model/src/kinds/index.ts

@@ -50,11 +50,6 @@ export type {
   SystemEntityV1alpha1 as SystemEntity,
   SystemEntityV1alpha1,
 } from './SystemEntityV1alpha1';
-export { templateEntityV1alpha1Validator } from './TemplateEntityV1alpha1';
-export type {
-  TemplateEntityV1alpha1 as TemplateEntity,
-  TemplateEntityV1alpha1,
-} from './TemplateEntityV1alpha1';
 export { templateEntityV1beta2Validator } from './TemplateEntityV1beta2';
 export type { TemplateEntityV1beta2 } from './TemplateEntityV1beta2';
 export type { KindValidator } from './types';

packages/catalog-model/src/schema/kinds/Template.v1alpha1.schema.json

@@ -1,99 +0,0 @@
-{
-  "$schema": "http://json-schema.org/draft-07/schema",
-  "$id": "TemplateV1alpha1",
-  "description": "A Template describes a skeleton for use with the Scaffolder. It is used for describing what templating library is supported, and also for documenting the variables that the template requires using JSON Forms Schema.",
-  "examples": [
-    {
-      "apiVersion": "backstage.io/v1alpha1",
-      "kind": "Template",
-      "metadata": {
-        "name": "react-ssr-template",
-        "title": "React SSR Template",
-        "description": "Next.js application skeleton for creating isomorphic web applications.",
-        "tags": ["recommended", "react"]
-      },
-      "spec": {
-        "owner": "artist-relations-team",
-        "templater": "cookiecutter",
-        "type": "website",
-        "path": ".",
-        "schema": {
-          "required": ["component-id", "description"],
-          "properties": {
-            "component_id": {
-              "title": "Name",
-              "type": "string",
-              "description": "Unique name of the component"
-            },
-            "description": {
-              "title": "Description",
-              "type": "string",
-              "description": "Description of the component"
-            }
-          }
-        }
-      }
-    }
-  ],
-  "allOf": [
-    {
-      "$ref": "Entity"
-    },
-    {
-      "type": "object",
-      "required": ["spec"],
-      "properties": {
-        "apiVersion": {
-          "enum": ["backstage.io/v1alpha1", "backstage.io/v1beta1"]
-        },
-        "kind": {
-          "enum": ["Template"]
-        },
-        "metadata": {
-          "type": "object",
-          "properties": {
-            "title": {
-              "type": "string",
-              "description": "The nice display name for the template. This field is required as is used to reference the template to the user instead of the metadata.name field.",
-              "examples": ["React SSR Template"],
-              "minLength": 1
-            }
-          }
-        },
-        "spec": {
-          "type": "object",
-          "required": ["type", "templater", "schema"],
-          "properties": {
-            "type": {
-              "type": "string",
-              "description": "The type of component created by the template. The software catalog accepts any type value, but an organization should take great care to establish a proper taxonomy for these. Tools including Backstage itself may read this field and behave differently depending on its value. For example, a website type component may present tooling in the Backstage interface that is specific to just websites.",
-              "examples": ["service", "website", "library"],
-              "minLength": 1
-            },
-            "templater": {
-              "type": "string",
-              "description": "The templating library that is supported by the template skeleton.",
-              "examples": ["cookiecutter"],
-              "minLength": 1
-            },
-            "path": {
-              "type": "string",
-              "description": "The string location where the templater should be run if it is not on the same level as the template.yaml definition.",
-              "examples": ["./cookiecutter/skeleton"],
-              "minLength": 1
-            },
-            "schema": {
-              "type": "object",
-              "description": "The JSONSchema describing the inputs for the template."
-            },
-            "owner": {
-              "type": "string",
-              "description": "The user (or group) owner of the template",
-              "minLength": 1
-            }
-          }
-        }
-      }
-    }
-  ]
-}

packages/catalog-model/src/validation/KubernetesValidatorFunctions.test.ts

@@ -76,9 +76,10 @@ describe('KubernetesValidatorFunctions', () => {
     ['a-b', true],
     ['-a-b', false],
     ['a-b-', false],
-    ['a--b', false],
+    ['a--b', true],
     ['a_b', true],
     ['a.b', true],
+    ['a..b', true],
   ])(`isValidObjectName %p ? %p`, (value, matches) => {
     expect(KubernetesValidatorFunctions.isValidObjectName(value)).toBe(matches);
   });
@@ -114,9 +115,10 @@ describe('KubernetesValidatorFunctions', () => {
     ['a-b', true],
     ['-a-b', false],
     ['a-b-', false],
-    ['a--b', false],
+    ['a--b', true],
     ['a_b', true],
     ['a.b', true],
+    ['a..b', true],
     ['a/a', true],
     ['a-b.c/a', true],
     ['a--b.c/a', false],
@@ -150,9 +152,10 @@ describe('KubernetesValidatorFunctions', () => {
     ['a-b', true],
     ['-a-b', false],
     ['a-b-', false],
-    ['a--b', false],
+    ['a--b', true],
     ['a_b', true],
     ['a.b', true],
+    ['a..b', true],
   ])(`isValidLabelValue %p ? %p`, (value, matches) => {
     expect(KubernetesValidatorFunctions.isValidLabelValue(value)).toBe(matches);
   });
@@ -169,9 +172,10 @@ describe('KubernetesValidatorFunctions', () => {
     ['a-b', true],
     ['-a-b', false],
     ['a-b-', false],
-    ['a--b', false],
+    ['a--b', true],
     ['a_b', true],
     ['a.b', true],
+    ['a..b', true],
     ['a/a', true],
     ['a-b.c/a', true],
     ['a--b.c/a', false],

packages/catalog-model/src/validation/KubernetesValidatorFunctions.ts

@@ -48,7 +48,7 @@ export class KubernetesValidatorFunctions {
       typeof value === 'string' &&
       value.length >= 1 &&
       value.length <= 63 &&
-      /^[a-z0-9A-Z]+([-_.][a-z0-9A-Z]+)*$/.test(value)
+      /^([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$/.test(value)
     );
   }