diff --git a/.changeset/bright-icons-repair.md b/.changeset/bright-icons-repair.md deleted file mode 100644 index 8c3f43403a..0000000000 --- a/.changeset/bright-icons-repair.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@backstage/plugin-kubernetes-backend': patch ---- - -Support HTTP 400 Bad Request from Kubernetes API diff --git a/.changeset/bright-radios-eat.md b/.changeset/bright-radios-eat.md deleted file mode 100644 index 73d2b32e23..0000000000 --- a/.changeset/bright-radios-eat.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@backstage/plugin-github-actions': minor ---- - -Support GHE diff --git a/.changeset/calm-turtles-smoke.md b/.changeset/calm-turtles-smoke.md deleted file mode 100644 index f2d473fe9f..0000000000 --- a/.changeset/calm-turtles-smoke.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -'@backstage/core': minor ---- - -Removed `InfoCard` variant `height100`, originally deprecated in [#2826](https://github.com/backstage/backstage/pull/2826). - -If your component still relies on this variant, simply replace it with `gridItem`. diff --git a/.changeset/clever-timers-thank.md b/.changeset/clever-timers-thank.md deleted file mode 100644 index b5b75c0409..0000000000 --- a/.changeset/clever-timers-thank.md +++ /dev/null @@ -1,25 +0,0 @@ ---- -'@backstage/backend-common': patch -'@backstage/integration': patch ---- - -Add support for GitHub Apps authentication for backend plugins. - -`GithubCredentialsProvider` requests and caches GitHub credentials based on a repository or organization url. - -The `GithubCredentialsProvider` class should be considered stateful since tokens will be cached internally. -Consecutive calls to get credentials will return the same token, tokens older than 50 minutes will be considered expired and reissued. -`GithubCredentialsProvider` will default to the configured access token if no GitHub Apps are configured. - -More information on how to create and configure a GitHub App to use with backstage can be found in the documentation. - -Usage: - -```javascript -const credentialsProvider = new GithubCredentialsProvider(config); -const { token, headers } = await credentialsProvider.getCredentials({ - url: 'https://github.com/', -}); -``` - -Updates `GithubUrlReader` to use the `GithubCredentialsProvider`. diff --git a/.changeset/cost-insights-careless-coins-pretend.md b/.changeset/cost-insights-careless-coins-pretend.md deleted file mode 100644 index 6314a7c60f..0000000000 --- a/.changeset/cost-insights-careless-coins-pretend.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@backstage/plugin-cost-insights': patch ---- - -bug(cost-insights): Remove entity count when none present diff --git a/.changeset/cost-insights-curvy-dingos-live.md b/.changeset/cost-insights-curvy-dingos-live.md new file mode 100644 index 0000000000..f15d3b0492 --- /dev/null +++ b/.changeset/cost-insights-curvy-dingos-live.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-cost-insights': patch +--- + +Allow expand functionality to top panel product chart tooltip. diff --git a/.changeset/green-coins-confess.md b/.changeset/green-coins-confess.md deleted file mode 100644 index 7fc52b8a43..0000000000 --- a/.changeset/green-coins-confess.md +++ /dev/null @@ -1,40 +0,0 @@ ---- -'@backstage/create-app': patch ---- - -Migrate to using `FlatRoutes` from `@backstage/core` for the root app routes. - -This is the first step in migrating applications as mentioned here: https://backstage.io/docs/plugins/composability#porting-existing-apps. - -To apply this change to an existing app, switch out the `Routes` component from `react-router` to `FlatRoutes` from `@backstage/core`. -This also allows you to remove any `/*` suffixes on the route paths. For example: - -```diff -import { - OAuthRequestDialog, - SidebarPage, - createRouteRef, -+ FlatRoutes, - } from '@backstage/core'; - import { AppSidebar } from './sidebar'; --import { Route, Routes, Navigate } from 'react-router'; -+import { Route, Navigate } from 'react-router'; - import { Router as CatalogRouter } from '@backstage/plugin-catalog'; -... - -- -+ -... - } - /> -- } /> -+ } /> -... - } /> -- -+ - -``` diff --git a/.changeset/healthy-comics-drive.md b/.changeset/healthy-comics-drive.md deleted file mode 100644 index 4b532dc853..0000000000 --- a/.changeset/healthy-comics-drive.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -'@backstage/plugin-catalog': patch ---- - -Display the owner, system, and domain as links to the entity pages in the about card. -Only display fields in the about card that are applicable to the entity kind. diff --git a/.changeset/healthy-crews-remember.md b/.changeset/healthy-crews-remember.md deleted file mode 100644 index c7cfdaaa6d..0000000000 --- a/.changeset/healthy-crews-remember.md +++ /dev/null @@ -1,19 +0,0 @@ ---- -'@backstage/create-app': patch ---- - -fix routing and config for user-settings plugin - -To make the corresponding change in your local app, add the following in your App.tsx - -``` -import { Router as SettingsRouter } from '@backstage/plugin-user-settings'; -... -} /> -``` - -and the following to your plugins.ts: - -``` -export { plugin as UserSettings } from '@backstage/plugin-user-settings'; -``` diff --git a/.changeset/honest-jokes-rush.md b/.changeset/honest-jokes-rush.md new file mode 100644 index 0000000000..31ec6202da --- /dev/null +++ b/.changeset/honest-jokes-rush.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-auth-backend': patch +--- + +Use .text instead of .json for ALB key response diff --git a/.changeset/loud-kids-dance.md b/.changeset/loud-kids-dance.md deleted file mode 100644 index 4fd32ac348..0000000000 --- a/.changeset/loud-kids-dance.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@backstage/cli': patch ---- - -Append `-credentials.yaml` to credentials file generated by `backstage-cli create-github-app` and display warning about sensitive contents. diff --git a/.changeset/orange-avocados-work.md b/.changeset/orange-avocados-work.md deleted file mode 100644 index a1f3597302..0000000000 --- a/.changeset/orange-avocados-work.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@backstage/plugin-catalog-backend': patch ---- - -Refuse to remove the bootstrap location diff --git a/.changeset/rich-turtles-roll.md b/.changeset/rich-turtles-roll.md deleted file mode 100644 index 9015161369..0000000000 --- a/.changeset/rich-turtles-roll.md +++ /dev/null @@ -1,19 +0,0 @@ ---- -'@backstage/backend-common': minor ---- - -Remove fallback option from `UrlReaders.create` and `UrlReaders.default`, as well as the default fallback reader. - -To be able to read data from endpoints outside of the configured integrations, you now need to explicitly allow it by -adding an entry in the `backend.reading.allow` list. For example: - -```yml -backend: - baseUrl: ... - reading: - allow: - - host: example.com - - host: '*.examples.org' -``` - -Apart from adding the above configuration, most projects should not need to take any action to migrate existing code. If you do happen to have your own fallback reader configured, this needs to be replaced with a reader factory that selects a specific set of URLs to work with. If you where wrapping the existing fallback reader, the new one that handles the allow list is created using `FetchUrlReader.factory`. diff --git a/.changeset/spoon-fork.md b/.changeset/spoon-fork.md deleted file mode 100644 index 5b8620f94e..0000000000 --- a/.changeset/spoon-fork.md +++ /dev/null @@ -1,12 +0,0 @@ ---- -'@backstage/create-app': patch ---- - -Add `*-credentials.yaml` to gitignore to prevent accidental commits of sensitive credential information. - -To apply this change to an existing installation, add these lines to your `.gitignore` - -```gitignore -# Sensitive credentials -*-credentials.yaml -``` diff --git a/.changeset/spotty-moons-tap.md b/.changeset/spotty-moons-tap.md deleted file mode 100644 index facb76a296..0000000000 --- a/.changeset/spotty-moons-tap.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@backstage/plugin-catalog-backend': patch ---- - -Reduce log noise on locations refresh diff --git a/.changeset/strong-ligers-lay.md b/.changeset/strong-ligers-lay.md deleted file mode 100644 index 56f4a91a9d..0000000000 --- a/.changeset/strong-ligers-lay.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -'@backstage/plugin-catalog': patch ---- - -Display systems in catalog table and make both owner and system link to the entity pages. -The owner field is now taken from the relations of the entity instead of its spec. diff --git a/.changeset/swift-baboons-refuse.md b/.changeset/swift-baboons-refuse.md deleted file mode 100644 index 4cefb534ff..0000000000 --- a/.changeset/swift-baboons-refuse.md +++ /dev/null @@ -1,53 +0,0 @@ ---- -'@backstage/create-app': patch ---- - -use `fromConfig` for all scaffolder helpers, and use the url protocol for app-config location entries. - -To apply this change to your local installation, replace the contents of your `packages/backend/src/plugins/scaffolder.ts` with the following contents: - -```ts -import { - CookieCutter, - createRouter, - Preparers, - Publishers, - CreateReactAppTemplater, - Templaters, - CatalogEntityClient, -} from '@backstage/plugin-scaffolder-backend'; -import { SingleHostDiscovery } from '@backstage/backend-common'; -import type { PluginEnvironment } from '../types'; -import Docker from 'dockerode'; - -export default async function createPlugin({ - logger, - config, -}: PluginEnvironment) { - const cookiecutterTemplater = new CookieCutter(); - const craTemplater = new CreateReactAppTemplater(); - const templaters = new Templaters(); - templaters.register('cookiecutter', cookiecutterTemplater); - templaters.register('cra', craTemplater); - - const preparers = await Preparers.fromConfig(config, { logger }); - const publishers = await Publishers.fromConfig(config, { logger }); - - const dockerClient = new Docker(); - - const discovery = SingleHostDiscovery.fromConfig(config); - const entityClient = new CatalogEntityClient({ discovery }); - - return await createRouter({ - preparers, - templaters, - publishers, - logger, - config, - dockerClient, - entityClient, - }); -} -``` - -This will ensure that the `scaffolder-backend` package can add handlers for the `url` protocol which is becoming the standard when registering entities in the `catalog` diff --git a/.changeset/techdocs-glasses-wonder.md b/.changeset/techdocs-glasses-wonder.md deleted file mode 100644 index 7610be2665..0000000000 --- a/.changeset/techdocs-glasses-wonder.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@backstage/techdocs-common': patch ---- - -TechDocs backend now streams files through from Google Cloud Storage to the browser, improving memory usage. diff --git a/.changeset/unlucky-cougars-grin.md b/.changeset/unlucky-cougars-grin.md deleted file mode 100644 index 3c48a612e4..0000000000 --- a/.changeset/unlucky-cougars-grin.md +++ /dev/null @@ -1,9 +0,0 @@ ---- -'@backstage/create-app': patch ---- - -Remove the `@types/helmet` dev dependency from the app template. This -dependency is now unused as the package `helmet` brings its own types. - -To update your existing app, simply remove the `@types/helmet` dependency from -the `package.json` of your backend package. diff --git a/.changeset/warm-months-bake.md b/.changeset/warm-months-bake.md deleted file mode 100644 index c8f3723eb1..0000000000 --- a/.changeset/warm-months-bake.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@backstage/plugin-lighthouse': patch ---- - -Fix display of floating point precision errors in card category scores diff --git a/.changeset/wild-cats-end.md b/.changeset/wild-cats-end.md deleted file mode 100644 index d2c9e6f245..0000000000 --- a/.changeset/wild-cats-end.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@backstage/plugin-catalog-backend': patch ---- - -Update the @azure/msal-node dependency to 1.0.0-beta.3. diff --git a/.changeset/wise-mice-invite.md b/.changeset/wise-mice-invite.md deleted file mode 100644 index 9021336c07..0000000000 --- a/.changeset/wise-mice-invite.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@backstage/backend-common': minor ---- - -Remove support for HTTPS certificate generation parameters. Use `backend.https = true` instead. diff --git a/.changeset/yellow-ties-switch.md b/.changeset/yellow-ties-switch.md deleted file mode 100644 index 784d2a18ae..0000000000 --- a/.changeset/yellow-ties-switch.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@backstage/catalog-model': minor ---- - -The catalog no longer attempts to merge old and new annotations, when updating an entity from a remote location. This was a behavior that was copied from kubernetes, and catered to use cases where you wanted to use HTTP POST to update an entity in-place, outside of what the refresh loop does. This has proved to be a mistake, because as a side effect, the refresh loop effectively is unable to ever delete annotations when they are removed from source YAML. This is obviously a breaking change, but we believe that this is not a behavior that is relied upon in the wild, and it has never been an actually supported use flow of the catalog. We therefore choose to break the behavior outright, and instead just store updated annotations verbatim - just like we already do for example for labels diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 11a756a597..9a5ef09554 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -6,6 +6,7 @@ * @backstage/maintainers /docs/features/techdocs @backstage/techdocs-core +/docs/features/search @backstage/techdocs-core /plugins/cost-insights @backstage/silver-lining /plugins/cloudbuild @trivago/ebarrios /plugins/search @backstage/techdocs-core diff --git a/.github/styles/vocab.txt b/.github/styles/vocab.txt index 28b0ec45d2..7e3867e31e 100644 --- a/.github/styles/vocab.txt +++ b/.github/styles/vocab.txt @@ -67,8 +67,10 @@ Dominik dtuite dzolotusky Ek +etag env Env +esbuild eslint Expedia facto @@ -84,6 +86,7 @@ GitHub GitLab Grafana GraphQL +graphql graphviz Gustavsson Hackathons @@ -92,6 +95,7 @@ Henneke Heroku horizontalpodautoscalers Hostname +hotspots html http https diff --git a/.github/workflows/fossa.yml b/.github/workflows/fossa.yml new file mode 100644 index 0000000000..f3fe2bbc5f --- /dev/null +++ b/.github/workflows/fossa.yml @@ -0,0 +1,44 @@ +name: FOSSA +on: + push: + branches: [master] + pull_request: + branches: [master] + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v2.3.4 + + # We use this to modify the generated .fossa.yml + - name: Install yq + run: sudo snap install yq + + - name: Install Fossa + run: "curl -H 'Cache-Control: no-cache' https://raw.githubusercontent.com/fossas/fossa-cli/master/install.sh | bash" + + - name: Configure Fossa + # The --option flag for fossa init does not work yet, see https://github.com/fossas/fossa-cli/issues/614 + run: | + fossa init + yq eval -i '.analyze.modules[].options.strategy = "yarn-list"' .fossa.yml + + # This deletes entries for template and example packages found within packages and plugins + # Seems like yq has a bug that causes only a subset of all matches to be deleted each run + yq eval -i 'del(.analyze.modules[] | select(.path == "*/*/**"))' .fossa.yml + yq eval -i 'del(.analyze.modules[] | select(.path == "*/*/**"))' .fossa.yml + yq eval -i 'del(.analyze.modules[] | select(.path == "*/*/**"))' .fossa.yml + yq eval -i 'del(.analyze.modules[] | select(.path == "*/*/**"))' .fossa.yml + yq eval -i 'del(.analyze.modules[] | select(.path == "*/*/**"))' .fossa.yml + + - name: Show config + run: cat .fossa.yml + + - name: Fossa Analyze + env: + # FOSSA Push-Only API Token + FOSSA_API_KEY: 9ee7e8893660832a7387dcc32377fb61 + run: fossa analyze --branch "$GITHUB_REF" diff --git a/app-config.yaml b/app-config.yaml index 741991877e..02afb8ac41 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -185,33 +185,46 @@ catalog: # groupFilter: securityEnabled eq false and mailEnabled eq true and groupTypes/any(c:c+eq+'Unified') locations: + # Add a location here to ingest it, for example from an URL: + # + # - type: url + # target: https://github.com/backstage/backstage/blob/master/packages/catalog-model/examples/all-components.yaml + # + # For local development you can use a file location instead: + # + # - type: file + # target: ../catalog-model/examples/all-components.yaml + # + # File locations are relative to the current working directory of the + # backend, for example packages/backend/. + # Backstage example components - - type: url - target: https://github.com/backstage/backstage/blob/master/packages/catalog-model/examples/all-components.yaml + - type: file + target: ../catalog-model/examples/all-components.yaml # Example component for github-actions - - type: url - target: https://github.com/backstage/backstage/blob/master/plugins/github-actions/examples/sample.yaml + - type: file + target: ../../plugins/github-actions/examples/sample.yaml # Example component for TechDocs - - type: url - target: https://github.com/backstage/backstage/blob/master/plugins/techdocs-backend/examples/documented-component/catalog-info.yaml + - type: file + target: ../../plugins/techdocs-backend/examples/documented-component/catalog-info.yaml # Backstage example APIs - - type: url - target: https://github.com/backstage/backstage/blob/master/packages/catalog-model/examples/all-apis.yaml + - type: file + target: ../catalog-model/examples/all-apis.yaml # Backstage example resources - - type: url - target: https://github.com/backstage/backstage/blob/master/packages/catalog-model/examples/all-resources.yaml + - type: file + target: ../catalog-model/examples/all-resources.yaml # Backstage example systems - - type: url - target: https://github.com/backstage/backstage/blob/master/packages/catalog-model/examples/all-systems.yaml + - type: file + target: ../catalog-model/examples/all-systems.yaml # Backstage example domains - - type: url - target: https://github.com/backstage/backstage/blob/master/packages/catalog-model/examples/all-domains.yaml + - type: file + target: ../catalog-model/examples/all-domains.yaml # Backstage example templates - - type: url - target: https://github.com/backstage/backstage/blob/master/plugins/scaffolder-backend/sample-templates/all-templates.yaml + - type: file + target: ../../plugins/scaffolder-backend/sample-templates/all-templates.yaml # Backstage example groups and users - - type: url - target: https://github.com/backstage/backstage/blob/master/packages/catalog-model/examples/acme-corp.yaml + - type: file + target: ../catalog-model/examples/acme-corp.yaml scaffolder: github: @@ -359,3 +372,8 @@ homepage: timezone: 'Asia/Tokyo' pagerduty: eventsBaseUrl: 'https://events.pagerduty.com/v2' + +kafka: + clientId: backstage + brokers: + - localhost:9092 diff --git a/catalog-info.yaml b/catalog-info.yaml index 2144d1709c..7e60af5755 100644 --- a/catalog-info.yaml +++ b/catalog-info.yaml @@ -6,7 +6,7 @@ metadata: Backstage is an open-source developer portal that puts the developer experience first. annotations: github.com/project-slug: backstage/backstage - backstage.io/techdocs-ref: url:https://github.com/backstage/backstage/tree/master + backstage.io/techdocs-ref: url:https://github.com/backstage/backstage lighthouse.com/website-url: https://backstage.io spec: type: library diff --git a/contrib/docs/tutorials/authenticate-api-requests.md b/contrib/docs/tutorials/authenticate-api-requests.md new file mode 100644 index 0000000000..2b8407abd2 --- /dev/null +++ b/contrib/docs/tutorials/authenticate-api-requests.md @@ -0,0 +1,52 @@ +# Authenticate API requests + +The Backstage backend APIs are by default available without authentication. To avoid evil-doers from accessing or modifying data, one might use a network protection mechanism such as a firewall or an authenticating reverse proxy. For Backstage instances that are available on the Internet one can instead use the experimental IdentityClient as outlined below. + +API requests from frontend plugins include an authorization header with a Backstage identity token acquired when the user logs in. By adding a middleware that verifies said token to be valid and signed by Backstage, non-authenticated requests can be blocked with a 401 Unauthorized response. + +Note that this means Backstage will stop working for guests, as no token is issued for them. + +Caveat: as of writing this, Backstage does not refresh the identity token so eventually users will get a 401 response on API calls (not on loading the web page as only the API calls are authenticated) and have to logout/login again to get a new token. + +```typescript +// packages/backend/src/index.ts from a create-app deployment + +import { Request, Response, NextFunction } from 'express'; +import { IdentityClient } from '@backstage/plugin-auth-backend'; + +// ... + +async function main() { + // ... + + const discovery = SingleHostDiscovery.fromConfig(config); + const identity = new IdentityClient({ + discovery, + issuer: await discovery.getExternalBaseUrl('auth'), + }); + const authMiddleware = async ( + req: Request, + res: Response, + next: NextFunction, + ) => { + try { + const token = IdentityClient.getBearerToken(req.headers.authorization); + req.user = await identity.authenticate(token); + next(); + } catch (error) { + res.status(401).send(`Unauthorized`); + } + }; + + const apiRouter = Router(); + // The auth route must be publically available as it is used during login + apiRouter.use('/auth', await auth(authEnv)); + // Only authenticated requests are allowed to the routes below + apiRouter.use('/catalog', authMiddleware, await catalog(catalogEnv)); + apiRouter.use('/techdocs', authMiddleware, await techdocs(techdocsEnv)); + apiRouter.use('/proxy', authMiddleware, await proxy(proxyEnv)); + apiRouter.use(authMiddleware, notFoundHandler()); + + // ... +} +``` diff --git a/docs/architecture-decisions/adr006-avoid-react-fc.md b/docs/architecture-decisions/adr006-avoid-react-fc.md index 51dcf042da..eecb4ad87a 100644 --- a/docs/architecture-decisions/adr006-avoid-react-fc.md +++ b/docs/architecture-decisions/adr006-avoid-react-fc.md @@ -43,6 +43,15 @@ const GoodComponent = ({ text, children }: GoodProps) => ( {children} ); + +/* Or as a shorthand, if no specifc child type is required */ +type GoodProps = PropsWithChildren<{ text: string }>; +const GoodComponent = ({ text, children }: GoodProps) => ( +
+
{text}
+ {children} +
+); ``` ## Consequences diff --git a/docs/features/software-catalog/descriptor-format.md b/docs/features/software-catalog/descriptor-format.md index 5b7c82152d..2b173c4b3e 100644 --- a/docs/features/software-catalog/descriptor-format.md +++ b/docs/features/software-catalog/descriptor-format.md @@ -977,8 +977,6 @@ metadata: spec: owner: artist-relations-team domain: artists - providesApis: - - artist-api ``` In addition to the [common envelope metadata](#common-to-all-kinds-the-metadata) diff --git a/docs/features/software-catalog/well-known-annotations.md b/docs/features/software-catalog/well-known-annotations.md index 3627c74b9a..2cbb829554 100644 --- a/docs/features/software-catalog/well-known-annotations.md +++ b/docs/features/software-catalog/well-known-annotations.md @@ -22,7 +22,7 @@ use. # Example: metadata: annotations: - backstage.io/managed-by-location: github:http://github.com/backstage/backstage/catalog-info.yaml + backstage.io/managed-by-location: url:http://github.com/backstage/backstage/blob/master/catalog-info.yaml ``` The value of this annotation is a so called location reference string, that @@ -30,8 +30,8 @@ points to the source from which the entity was originally fetched. This annotation is added automatically by the catalog as it fetches the data from a registered location, and is not meant to normally be written by humans. The annotation may point to any type of generic location that the catalog supports, -so it cannot be relied on to always be specifically of type `github`, nor that -it even represents a single file. Note also that a single location can be the +so it cannot be relied on to always be specifically of type `url`, nor that it +even represents a single file. Note also that a single location can be the source of many entities, so it represents a many-to-one relationship. The format of the value is `:`. Note that the target may also @@ -40,13 +40,30 @@ expecting a two-item array out of it. The format of the target part is type-dependent and could conceivably even be an empty string, but the separator colon is always present. +### backstage.io/managed-by-origin-location + +```yaml +# Example: +metadata: + annotations: + backstage.io/managed-by-origin-location: url:http://github.com/backstage/backstage/blob/master/catalog-info.yaml +``` + +The value of this annotation is a location reference string (see above). It +points to the location, whose registration lead to the creation of the entity. +In most cases, the `backstage.io/managed-by-location` and +`backstage.io/managed-by-origin-location` will be equal. They will be different +if the original location delegates to another location. A common case is, that a +location is registered as `bootstrap:bootstrap` which means that it is part of +the `app-config.yaml` of a Backstage installation. + ### backstage.io/techdocs-ref ```yaml # Example: metadata: annotations: - backstage.io/techdocs-ref: github:https://github.com/backstage/backstage.git + backstage.io/techdocs-ref: url:https://github.com/backstage/backstage/tree/master ``` The value of this annotation is a location reference string (see above). If this diff --git a/docs/features/software-templates/extending/create-your-own-publisher.md b/docs/features/software-templates/extending/create-your-own-publisher.md index 97854c999a..5e2809e006 100644 --- a/docs/features/software-templates/extending/create-your-own-publisher.md +++ b/docs/features/software-templates/extending/create-your-own-publisher.md @@ -57,7 +57,7 @@ That type looks like the following: export type PublisherBase = { publish(opts: { entity: TemplateEntityV1alpha1; - values: RequiredTemplateValues & Record; + values: TemplaterValues; directory: string; }): Promise<{ remoteUrl: string }>; }; diff --git a/docs/features/software-templates/extending/create-your-own-templater.md b/docs/features/software-templates/extending/create-your-own-templater.md index 19c54733b4..37e68c4153 100644 --- a/docs/features/software-templates/extending/create-your-own-templater.md +++ b/docs/features/software-templates/extending/create-your-own-templater.md @@ -61,7 +61,7 @@ That type looks like the following: ```ts export type TemplaterRunOptions = { directory: string; - values: RequiredTemplateValues & Record; + values: TemplaterValues; logStream?: Writable; dockerClient: Docker; }; diff --git a/docs/features/techdocs/architecture.md b/docs/features/techdocs/architecture.md index 53149ae4f3..92cd78b071 100644 --- a/docs/features/techdocs/architecture.md +++ b/docs/features/techdocs/architecture.md @@ -123,6 +123,20 @@ a cache for the generated static content. TechDocs is also currently built on MkDocs which does not allow us to generate docs per-page, so we would have to build all docs for a entity on every request. +**Q. Can you use the techdocs plugin without the techdocs-backend plugin?** + +A: `techdocs` and `techdocs-backend` plugins are designed to be used together, +like any other Backstage plugin with a frontend and its backend (catalog, +scaffolder, etc.). If you set your Backstage instance to generate docs on the +server, `techdocs-backend` will be responsible for managing the whole build +process, making sure it's scalable. It is responsible for securely communicating +with the cloud storage provider, for both fetching static generated sites and +publishing the updates. There are other planned features like an authentication +layer for users to determine whether they have the permission to view a +particular docs site. There are a handful of features which are extremely hard +to develop without a tightly integrated backend in place. Hence, support for +`techdocs` without `techdocs-backend` is limited and challenging to develop. + # Future work _Ideas here are far fetched and not in the project's milestone for near future diff --git a/docs/overview/roadmap.md b/docs/overview/roadmap.md index 6ed4a3272d..cd319c4ade 100644 --- a/docs/overview/roadmap.md +++ b/docs/overview/roadmap.md @@ -57,8 +57,10 @@ guidelines to get started. see and manage their services running in K8s, regardless if that's locally, in AWS, GCS, Azure, or elsewhere. -- **Global search** - Extend the basic search functionality currently available - in the Backstage Service Catalog to become a global search experience. +- **[Search platform](../features/search/README.md)** - Evolve the basic search + functionality currently available into a platform that **a)** enables search + across the software catalog, TechDocs, and any other information exposed by + plugins, and **b)** supports a variety of search engine technologies. - **[Software Templates V2](https://github.com/backstage/backstage/issues/2771)** - Expand the templates to make the steps more composable by adding the ability diff --git a/docs/plugins/plugin-development.md b/docs/plugins/plugin-development.md index c5d9763f7d..109f6ed6ba 100644 --- a/docs/plugins/plugin-development.md +++ b/docs/plugins/plugin-development.md @@ -54,13 +54,4 @@ addRoute( Component: ComponentType, options?: RouteOptions, ): void; - -/** - * @deprecated See the `addRoute` method - */ -registerRoute( - path: RoutePath, - Component: ComponentType, - options?: RouteOptions, -): void; ``` diff --git a/docs/reference/createPlugin-router.md b/docs/reference/createPlugin-router.md index 89ee44e558..0ef5bdbd0f 100644 --- a/docs/reference/createPlugin-router.md +++ b/docs/reference/createPlugin-router.md @@ -15,15 +15,6 @@ addRoute( Component: ComponentType, options?: RouteOptions, ): void; - -/** - * @deprecated See the `addRoute` method - */ -registerRoute( - path: RoutePath, - Component: ComponentType, - options?: RouteOptions, -): void; ``` ## RouteRef diff --git a/microsite/core/Components.js b/microsite/core/Components.js index 49e7d7daca..4811b49291 100644 --- a/microsite/core/Components.js +++ b/microsite/core/Components.js @@ -106,9 +106,29 @@ const Breakpoint = ({ narrow, wide }) => ( ); +const Banner = simpleComponent('div', 'Banner', ['hidden']); +Banner.Container = simpleComponent('div', 'Banner__Container'); + +const BannerDismissButton = simpleComponent( + props => ( + + + + ), + 'Banner__DismissButton', +); + +Banner.Dismissable = ({ storageKey, children }) => ( + +); + module.exports = { Block, ActionBlock, Breakpoint, BulletLine, + Banner, }; diff --git a/microsite/data/plugins/argo-cd.yaml b/microsite/data/plugins/argo-cd.yaml index a47fa08d82..ca753f682a 100644 --- a/microsite/data/plugins/argo-cd.yaml +++ b/microsite/data/plugins/argo-cd.yaml @@ -2,7 +2,7 @@ title: Argo CD author: roadie.io authorUrl: https://roadie.io -category: CI +category: CI/CD description: View Argo CD status for your projects in Backstage. documentation: https://roadie.io/backstage/plugins/argo-cd iconUrl: https://roadie.io/images/logos/argo.png diff --git a/microsite/data/plugins/buildkite.yaml b/microsite/data/plugins/buildkite.yaml index eeb9440aac..16c6960898 100644 --- a/microsite/data/plugins/buildkite.yaml +++ b/microsite/data/plugins/buildkite.yaml @@ -2,7 +2,7 @@ title: Buildkite author: roadie.io authorUrl: https://roadie.io -category: CI +category: CI/CD description: View Buildkite CI builds for your service in Backstage. documentation: https://roadie.io/backstage/plugins/buildkite iconUrl: https://roadie.io/images/logos/buildkite.png diff --git a/microsite/data/plugins/circleci.yaml b/microsite/data/plugins/circleci.yaml index 467da9fb1e..0acad832fa 100644 --- a/microsite/data/plugins/circleci.yaml +++ b/microsite/data/plugins/circleci.yaml @@ -2,7 +2,7 @@ title: CircleCI author: Spotify authorUrl: https://github.com/spotify -category: CI +category: CI/CD description: Automate your development process with CI hosted in the cloud or on a private server. documentation: https://github.com/backstage/backstage/tree/master/plugins/circleci iconUrl: https://www.saaves.com/storage/brochure/logo-circleci-icon1583764538.png diff --git a/microsite/data/plugins/cloud-build.yaml b/microsite/data/plugins/cloud-build.yaml index 2b40d02de2..253a935392 100644 --- a/microsite/data/plugins/cloud-build.yaml +++ b/microsite/data/plugins/cloud-build.yaml @@ -2,7 +2,7 @@ title: Google Cloud Build author: Trivago authorUrl: https://www.trivago.com -category: CI +category: CI/CD description: Build, test, and deploy on Google's serverless CI/CD platform. documentation: https://github.com/backstage/backstage/tree/master/plugins/cloudbuild iconUrl: https://avatars2.githubusercontent.com/u/38220399?s=400&v=4 diff --git a/microsite/data/plugins/gcp-projects.yaml b/microsite/data/plugins/gcp-projects.yaml index 1c015821b2..7c4b75d2a4 100644 --- a/microsite/data/plugins/gcp-projects.yaml +++ b/microsite/data/plugins/gcp-projects.yaml @@ -2,7 +2,7 @@ title: GCP Project Creator author: Trivago authorUrl: https://www.trivago.com -category: Cloud +category: Infrastructure description: Create, list and manage your Google Cloud Projects. documentation: https://github.com/backstage/backstage/tree/master/plugins/gcp-projects iconUrl: https://avatars1.githubusercontent.com/u/2810941?s=280&v=4 diff --git a/microsite/data/plugins/github-actions.yaml b/microsite/data/plugins/github-actions.yaml index aeccdc9b86..8d5a74d9ce 100644 --- a/microsite/data/plugins/github-actions.yaml +++ b/microsite/data/plugins/github-actions.yaml @@ -2,7 +2,7 @@ title: GitHub Actions author: Spotify authorUrl: https://github.com/spotify -category: CI +category: CI/CD description: GitHub Actions makes it easy to automate all your software workflows, now with world-class CI/CD. Build, test, and deploy your code right from GitHub. documentation: https://github.com/backstage/backstage/tree/master/plugins/github-actions iconUrl: https://avatars2.githubusercontent.com/u/44036562?s=400&v=4 diff --git a/microsite/data/plugins/github-insights.yaml b/microsite/data/plugins/github-insights.yaml index 1dba3dcfd1..f127f88a3c 100644 --- a/microsite/data/plugins/github-insights.yaml +++ b/microsite/data/plugins/github-insights.yaml @@ -2,7 +2,7 @@ title: GitHub Insights author: roadie.io authorUrl: https://roadie.io -category: Monitoring +category: Source Control Mgmt description: View GitHub Insights for your components in Backstage. documentation: https://roadie.io/backstage/plugins/github-insights iconUrl: https://roadie.io/images/logos/insights.png diff --git a/microsite/data/plugins/github-pull-requests.yaml b/microsite/data/plugins/github-pull-requests.yaml index 8e44a4bf56..4172cdff6b 100644 --- a/microsite/data/plugins/github-pull-requests.yaml +++ b/microsite/data/plugins/github-pull-requests.yaml @@ -2,7 +2,7 @@ title: GitHub Pull Requests author: roadie.io authorUrl: https://roadie.io/ -category: CI +category: Source Control Mgmt description: View GitHub pull requests for your service in Backstage. documentation: https://roadie.io/backstage/plugins/github-pull-requests iconUrl: https://roadie.io/images/logos/github.png diff --git a/microsite/data/plugins/jenkins.yaml b/microsite/data/plugins/jenkins.yaml index 638758d54d..7a8f884221 100644 --- a/microsite/data/plugins/jenkins.yaml +++ b/microsite/data/plugins/jenkins.yaml @@ -2,7 +2,7 @@ title: Jenkins author: '@timja' authorUrl: https://github.com/timja -category: CI +category: CI/CD description: Jenkins offers a simple way to set up a continuous integration and continuous delivery environment. documentation: https://github.com/backstage/backstage/tree/master/plugins/jenkins iconUrl: https://img.icons8.com/color/1600/jenkins.png diff --git a/microsite/data/plugins/jira.yaml b/microsite/data/plugins/jira.yaml index 579243d11c..c32455c7d5 100644 --- a/microsite/data/plugins/jira.yaml +++ b/microsite/data/plugins/jira.yaml @@ -2,7 +2,7 @@ title: Jira author: roadie.io authorUrl: https://roadie.io -category: Project Management +category: Agile Planning description: View Jira summary for your projects in Backstage. documentation: https://roadie.io/backstage/plugins/jira iconUrl: https://roadie.io/images/logos/jira.png diff --git a/microsite/data/plugins/kafka.yaml b/microsite/data/plugins/kafka.yaml new file mode 100644 index 0000000000..47d354b44f --- /dev/null +++ b/microsite/data/plugins/kafka.yaml @@ -0,0 +1,11 @@ +--- +title: Kafka +author: '@nirga' +authorUrl: https://github.com/nirga +category: Monitoring +description: Observability for Apache Kafka clusters and async API of components. +documentation: https://github.com/backstage/backstage/tree/master/plugins/kafka +iconUrl: https://kafka.apache.org/images/apache-kafka.png +npmPackageName: '@backstage/plugin-kafka' +tags: + - monitoring diff --git a/microsite/data/plugins/lighthouse.yaml b/microsite/data/plugins/lighthouse.yaml index 88d444e0ef..be70dfd74a 100644 --- a/microsite/data/plugins/lighthouse.yaml +++ b/microsite/data/plugins/lighthouse.yaml @@ -2,7 +2,7 @@ title: Lighthouse author: Spotify authorUrl: https://github.com/spotify -category: Accessibility +category: Quality description: Google's Lighthouse tool is a great resource for benchmarking and improving the accessibility, performance, SEO, and best practices of your website. documentation: https://github.com/backstage/backstage/tree/master/plugins/lighthouse iconUrl: https://seeklogo.com/images/G/google-lighthouse-logo-1C7FA08580-seeklogo.com.png diff --git a/microsite/data/plugins/snyk-security.yaml b/microsite/data/plugins/snyk-security.yaml new file mode 100644 index 0000000000..35f00595d1 --- /dev/null +++ b/microsite/data/plugins/snyk-security.yaml @@ -0,0 +1,9 @@ +--- +title: Snyk +author: Snyk Tech Services +authorUrl: https://snyk.io +category: Security +description: View Snyk scanned vulnerabilities and license compliance of your components directly in Backstage. +documentation: https://github.com/snyk-tech-services/backstage-plugin-snyk/blob/main/README.md +iconUrl: https://storage.googleapis.com/snyk-technical-services.appspot.com/snyk-logo-vertical-black.png +npmPackageName: 'backstage-plugin-snyk' diff --git a/microsite/data/plugins/travis-ci.yaml b/microsite/data/plugins/travis-ci.yaml index fe1b18c9d4..a6b8c6ebd7 100644 --- a/microsite/data/plugins/travis-ci.yaml +++ b/microsite/data/plugins/travis-ci.yaml @@ -2,7 +2,7 @@ title: Travis CI author: roadie.io authorUrl: https://roadie.io/ -category: CI +category: CI/CD description: View Travis CI builds for your service in Backstage. documentation: https://roadie.io/backstage/plugins/travis-ci iconUrl: https://roadie.io/images/logos/travis.png diff --git a/microsite/pages/en/index.js b/microsite/pages/en/index.js index d572605171..47f9568d9b 100644 --- a/microsite/pages/en/index.js +++ b/microsite/pages/en/index.js @@ -11,6 +11,7 @@ const Block = Components.Block; const ActionBlock = Components.ActionBlock; const Breakpoint = Components.Breakpoint; const BulletLine = Components.BulletLine; +const Banner = Components.Banner; class Index extends React.Component { render() { @@ -53,6 +54,15 @@ class Index extends React.Component { + + + 🎉 New feature: Kubernetes for service owners.{' '} + + Learn more. + + + + diff --git a/microsite/pages/en/plugins.js b/microsite/pages/en/plugins.js index c7a626bd1d..9ee54173f1 100644 --- a/microsite/pages/en/plugins.js +++ b/microsite/pages/en/plugins.js @@ -97,14 +97,14 @@ const Plugins = () => (

- See what plugins are already{' '} + See what plugins are already in progress - {' '} - and 👍. Missing a plugin for your favorite tool? Please{' '} + + and 👍. Missing a plugin for your favorite tool? Please suggest - {' '} + a new one.

diff --git a/microsite/siteConfig.js b/microsite/siteConfig.js index 39ba05bcad..8d0c4ea57c 100644 --- a/microsite/siteConfig.js +++ b/microsite/siteConfig.js @@ -82,6 +82,7 @@ const siteConfig = { 'https://buttons.github.io/buttons.js', 'https://unpkg.com/medium-zoom@1.0.6/dist/medium-zoom.min.js', '/js/medium-zoom.js', + '/js/dismissable-banner.js', ], // On page navigation for the current documentation page. diff --git a/microsite/static/css/custom.css b/microsite/static/css/custom.css index 01bb3c89eb..c947c88e66 100644 --- a/microsite/static/css/custom.css +++ b/microsite/static/css/custom.css @@ -1035,6 +1035,55 @@ code { } } +.Banner { + position: relative; + padding: 14px; + margin: 14px 20px; + border-radius: 4px; + background-color: $primaryColor; + font-family: Helvetica Neue, sans-serif; + color: #000; +} + +.Banner--hidden { + opacity: 0; + transition: opacity 200ms ease-in-out; +} + +.Banner a { + color: #000; + text-decoration: underline; +} + +.Banner__Container { + position: relative; + overflow: visible; + z-index: 100; + + max-width: 1430px; + height: 0; + margin: -14px auto 14px auto; +} + +.Banner__DismissButton { + position: absolute; + display: flex; + right: 8px; + top: 0; + bottom: 0; + margin: auto; + + border-radius: 50%; + padding: 6px; + width: 36px; + height: 36px; + cursor: pointer; +} + +.Banner__DismissButton:hover { + background: rgba(0, 0, 0, 0.2); +} + .logos-mobile-background { position: absolute; width: 200vw; diff --git a/microsite/static/js/dismissable-banner.js b/microsite/static/js/dismissable-banner.js new file mode 100644 index 0000000000..2fcdbe5692 --- /dev/null +++ b/microsite/static/js/dismissable-banner.js @@ -0,0 +1,18 @@ +window.addEventListener('DOMContentLoaded', () => { + const banners = document.querySelectorAll('[data-banner]'); + banners.forEach(banner => { + const storageKey = `hideBanner/${banner.getAttribute('data-banner')}`; + + if (!localStorage.getItem(storageKey)) { + banner.classList.remove('Banner--hidden'); + } + + const dismissButton = banner.querySelector('[data-banner-dismiss]'); + if (dismissButton) { + dismissButton.addEventListener('click', () => { + banner.classList.add('Banner--hidden'); + localStorage.setItem(storageKey, 'true'); + }); + } + }); +}); diff --git a/package.json b/package.json index aaa68e67de..758a8ba884 100644 --- a/package.json +++ b/package.json @@ -75,7 +75,7 @@ }, "jest": { "transformModules": [ - "@kyma-project/asyncapi-react" + "@asyncapi/react-component" ] } } diff --git a/packages/app/CHANGELOG.md b/packages/app/CHANGELOG.md index bd159afb9e..d6157f560f 100644 --- a/packages/app/CHANGELOG.md +++ b/packages/app/CHANGELOG.md @@ -1,5 +1,56 @@ # example-app +## 0.2.12 + +### Patch Changes + +- Updated dependencies [def2307f3] +- Updated dependencies [46bba09ea] +- Updated dependencies [efd6ef753] +- Updated dependencies [593632f07] +- Updated dependencies [8c2437c15] +- Updated dependencies [2b514d532] +- Updated dependencies [33846acfc] +- Updated dependencies [b604a9d41] +- Updated dependencies [d014185db] +- Updated dependencies [a187b8ad0] +- Updated dependencies [8855f61f6] +- Updated dependencies [ed6baab66] +- Updated dependencies [f04db53d7] +- Updated dependencies [a5e27d5c1] +- Updated dependencies [debf359b5] +- Updated dependencies [a93f42213] + - @backstage/catalog-model@0.7.0 + - @backstage/plugin-github-actions@0.3.0 + - @backstage/core@0.5.0 + - @backstage/plugin-catalog@0.2.12 + - @backstage/plugin-cost-insights@0.5.7 + - @backstage/plugin-catalog-import@0.3.5 + - @backstage/cli@0.4.7 + - @backstage/plugin-kubernetes@0.3.6 + - @backstage/plugin-api-docs@0.4.3 + - @backstage/plugin-scaffolder@0.4.0 + - @backstage/plugin-techdocs@0.5.4 + - @backstage/plugin-lighthouse@0.2.8 + - @backstage/plugin-circleci@0.2.6 + - @backstage/plugin-cloudbuild@0.2.7 + - @backstage/plugin-jenkins@0.3.6 + - @backstage/plugin-kafka@0.1.1 + - @backstage/plugin-org@0.3.4 + - @backstage/plugin-pagerduty@0.2.6 + - @backstage/plugin-register-component@0.2.7 + - @backstage/plugin-rollbar@0.2.8 + - @backstage/plugin-search@0.2.6 + - @backstage/plugin-sentry@0.3.3 + - @backstage/plugin-explore@0.2.3 + - @backstage/plugin-gcp-projects@0.2.3 + - @backstage/plugin-gitops-profiles@0.2.3 + - @backstage/plugin-graphiql@0.2.6 + - @backstage/plugin-newrelic@0.2.3 + - @backstage/plugin-tech-radar@0.3.3 + - @backstage/plugin-user-settings@0.2.4 + - @backstage/plugin-welcome@0.2.4 + ## 0.2.9 ### Patch Changes diff --git a/packages/app/package.json b/packages/app/package.json index e83af5a0c6..7734dfc244 100644 --- a/packages/app/package.json +++ b/packages/app/package.json @@ -1,38 +1,39 @@ { "name": "example-app", - "version": "0.2.9", + "version": "0.2.12", "private": true, "bundled": true, "dependencies": { - "@backstage/catalog-model": "^0.6.0", - "@backstage/cli": "^0.4.5", - "@backstage/core": "^0.4.2", - "@backstage/plugin-api-docs": "^0.4.1", - "@backstage/plugin-catalog": "^0.2.10", - "@backstage/plugin-catalog-import": "^0.3.3", - "@backstage/plugin-circleci": "^0.2.5", - "@backstage/plugin-cloudbuild": "^0.2.5", - "@backstage/plugin-cost-insights": "^0.5.5", - "@backstage/plugin-explore": "^0.2.2", - "@backstage/plugin-gcp-projects": "^0.2.2", - "@backstage/plugin-github-actions": "^0.2.6", - "@backstage/plugin-gitops-profiles": "^0.2.2", - "@backstage/plugin-graphiql": "^0.2.3", - "@backstage/plugin-org": "^0.3.2", - "@backstage/plugin-jenkins": "^0.3.4", - "@backstage/plugin-kubernetes": "^0.3.3", - "@backstage/plugin-lighthouse": "^0.2.6", - "@backstage/plugin-newrelic": "^0.2.2", - "@backstage/plugin-pagerduty": "0.2.5", - "@backstage/plugin-register-component": "^0.2.5", - "@backstage/plugin-rollbar": "^0.2.7", - "@backstage/plugin-scaffolder": "^0.3.6", - "@backstage/plugin-sentry": "^0.3.2", - "@backstage/plugin-search": "^0.2.5", - "@backstage/plugin-tech-radar": "^0.3.2", - "@backstage/plugin-techdocs": "^0.5.1", - "@backstage/plugin-user-settings": "^0.2.3", - "@backstage/plugin-welcome": "^0.2.3", + "@backstage/catalog-model": "^0.7.0", + "@backstage/cli": "^0.4.7", + "@backstage/core": "^0.5.0", + "@backstage/plugin-api-docs": "^0.4.3", + "@backstage/plugin-catalog": "^0.2.12", + "@backstage/plugin-catalog-import": "^0.3.5", + "@backstage/plugin-circleci": "^0.2.6", + "@backstage/plugin-cloudbuild": "^0.2.7", + "@backstage/plugin-cost-insights": "^0.5.7", + "@backstage/plugin-explore": "^0.2.3", + "@backstage/plugin-gcp-projects": "^0.2.3", + "@backstage/plugin-github-actions": "^0.3.0", + "@backstage/plugin-gitops-profiles": "^0.2.3", + "@backstage/plugin-graphiql": "^0.2.6", + "@backstage/plugin-org": "^0.3.4", + "@backstage/plugin-jenkins": "^0.3.6", + "@backstage/plugin-kafka": "^0.1.1", + "@backstage/plugin-kubernetes": "^0.3.6", + "@backstage/plugin-lighthouse": "^0.2.8", + "@backstage/plugin-newrelic": "^0.2.3", + "@backstage/plugin-pagerduty": "0.2.6", + "@backstage/plugin-register-component": "^0.2.7", + "@backstage/plugin-rollbar": "^0.2.8", + "@backstage/plugin-scaffolder": "^0.4.0", + "@backstage/plugin-sentry": "^0.3.3", + "@backstage/plugin-search": "^0.2.6", + "@backstage/plugin-tech-radar": "^0.3.3", + "@backstage/plugin-techdocs": "^0.5.4", + "@backstage/plugin-user-settings": "^0.2.4", + "@backstage/plugin-welcome": "^0.2.4", "@backstage/theme": "^0.2.2", "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", diff --git a/packages/app/src/components/catalog/EntityPage.tsx b/packages/app/src/components/catalog/EntityPage.tsx index a4d097b1d4..a1ab181211 100644 --- a/packages/app/src/components/catalog/EntityPage.tsx +++ b/packages/app/src/components/catalog/EntityPage.tsx @@ -63,6 +63,7 @@ import { UserProfileCard, } from '@backstage/plugin-org'; import { Router as SentryRouter } from '@backstage/plugin-sentry'; +import { Router as KafkaRouter } from '@backstage/plugin-kafka'; import { EmbeddedDocsRouter as DocsRouter } from '@backstage/plugin-techdocs'; import { Button, Grid } from '@material-ui/core'; import { @@ -243,6 +244,11 @@ const ServiceEntityPage = ({ entity }: { entity: Entity }) => ( title="Code Insights" element={} /> + } + /> ); diff --git a/packages/app/src/plugins.ts b/packages/app/src/plugins.ts index 6a4924cb0e..f07be07ec1 100644 --- a/packages/app/src/plugins.ts +++ b/packages/app/src/plugins.ts @@ -43,3 +43,4 @@ export { plugin as PagerDuty } from '@backstage/plugin-pagerduty'; export { plugin as Buildkite } from '@roadiehq/backstage-plugin-buildkite'; export { plugin as Search } from '@backstage/plugin-search'; export { plugin as Org } from '@backstage/plugin-org'; +export { plugin as Kafka } from '@backstage/plugin-kafka'; diff --git a/packages/backend-common/CHANGELOG.md b/packages/backend-common/CHANGELOG.md index 0814347c71..84265c9897 100644 --- a/packages/backend-common/CHANGELOG.md +++ b/packages/backend-common/CHANGELOG.md @@ -1,5 +1,79 @@ # @backstage/backend-common +## 0.5.0 + +### Minor Changes + +- 5345a1f98: Remove fallback option from `UrlReaders.create` and `UrlReaders.default`, as well as the default fallback reader. + + To be able to read data from endpoints outside of the configured integrations, you now need to explicitly allow it by + adding an entry in the `backend.reading.allow` list. For example: + + ```yml + backend: + baseUrl: ... + reading: + allow: + - host: example.com + - host: '*.examples.org' + ``` + + Apart from adding the above configuration, most projects should not need to take any action to migrate existing code. If you do happen to have your own fallback reader configured, this needs to be replaced with a reader factory that selects a specific set of URLs to work with. If you where wrapping the existing fallback reader, the new one that handles the allow list is created using `FetchUrlReader.factory`. + +- 09a370426: Remove support for HTTPS certificate generation parameters. Use `backend.https = true` instead. + +### Patch Changes + +- 0b135e7e0: Add support for GitHub Apps authentication for backend plugins. + + `GithubCredentialsProvider` requests and caches GitHub credentials based on a repository or organization url. + + The `GithubCredentialsProvider` class should be considered stateful since tokens will be cached internally. + Consecutive calls to get credentials will return the same token, tokens older than 50 minutes will be considered expired and reissued. + `GithubCredentialsProvider` will default to the configured access token if no GitHub Apps are configured. + + More information on how to create and configure a GitHub App to use with backstage can be found in the documentation. + + Usage: + + ```javascript + const credentialsProvider = new GithubCredentialsProvider(config); + const { token, headers } = await credentialsProvider.getCredentials({ + url: 'https://github.com/', + }); + ``` + + Updates `GithubUrlReader` to use the `GithubCredentialsProvider`. + +- 294a70cab: 1. URL Reader's `readTree` method now returns an `etag` in the response along with the blob. The etag is an identifier of the blob and will only change if the blob is modified on the target. Usually it is set to the latest commit SHA on the target. + + `readTree` also takes an optional `etag` in its options and throws a `NotModifiedError` if the etag matches with the etag of the resource. + + So, the `etag` can be used in building a cache when working with URL Reader. + + An example - + + ```ts + const response = await reader.readTree( + 'https://github.com/backstage/backstage', + ); + + const etag = response.etag; + + // Will throw a new NotModifiedError (exported from @backstage/backstage-common) + await reader.readTree('https://github.com/backstage/backstage', { + etag, + }); + ``` + + 2. URL Reader's readTree method can now detect the default branch. So, `url:https://github.com/org/repo/tree/master` can be replaced with `url:https://github.com/org/repo` in places like `backstage.io/techdocs-ref`. + +- 0ea032763: URL Reader: Use API response headers for archive filename in readTree. Fixes bug for users with hosted Bitbucket. +- Updated dependencies [0b135e7e0] +- Updated dependencies [fa8ba330a] +- Updated dependencies [ed6baab66] + - @backstage/integration@0.3.0 + ## 0.4.3 ### Patch Changes diff --git a/packages/backend-common/config.d.ts b/packages/backend-common/config.d.ts index 0e7561f264..74c199e737 100644 --- a/packages/backend-common/config.d.ts +++ b/packages/backend-common/config.d.ts @@ -109,81 +109,4 @@ export interface Config { */ csp?: { [policyId: string]: string[] | false }; }; - - /** Configuration for integrations towards various external repository provider systems */ - integrations?: { - /** Integration configuration for Azure */ - azure?: Array<{ - /** - * The hostname of the given Azure instance - */ - host: string; - /** - * Token used to authenticate requests. - * @visibility secret - */ - token?: string; - }>; - - /** Integration configuration for BitBucket */ - bitbucket?: Array<{ - /** - * The hostname of the given Bitbucket instance - */ - host: string; - /** - * Token used to authenticate requests. - * @visibility secret - */ - token?: string; - /** - * The base url for the BitBucket API, for example https://api.bitbucket.org/2.0 - */ - apiBaseUrl?: string; - /** - * The username to use for authenticated requests. - * @visibility secret - */ - username?: string; - /** - * BitBucket app password used to authenticate requests. - * @visibility secret - */ - appPassword?: string; - }>; - - /** Integration configuration for GitHub */ - github?: Array<{ - /** - * The hostname of the given GitHub instance - */ - host: string; - /** - * Token used to authenticate requests. - * @visibility secret - */ - token?: string; - /** - * The base url for the GitHub API, for example https://api.github.com - */ - apiBaseUrl?: string; - /** - * The base url for GitHub raw resources, for example https://raw.githubusercontent.com - */ - rawBaseUrl?: string; - }>; - - /** Integration configuration for GitLab */ - gitlab?: Array<{ - /** - * The hostname of the given GitLab instance - */ - host: string; - /** - * Token used to authenticate requests. - * @visibility secret - */ - token?: string; - }>; - }; } diff --git a/packages/backend-common/package.json b/packages/backend-common/package.json index c311a140f6..c698351615 100644 --- a/packages/backend-common/package.json +++ b/packages/backend-common/package.json @@ -1,7 +1,7 @@ { "name": "@backstage/backend-common", "description": "Common functionality library for Backstage backends", - "version": "0.4.3", + "version": "0.5.0", "main": "src/index.ts", "types": "src/index.ts", "private": false, @@ -32,7 +32,7 @@ "@backstage/cli-common": "^0.1.1", "@backstage/config": "^0.1.2", "@backstage/config-loader": "^0.4.1", - "@backstage/integration": "^0.2.0", + "@backstage/integration": "^0.3.0", "@types/cors": "^2.8.6", "@types/express": "^4.17.6", "archiver": "^5.0.2", @@ -66,7 +66,7 @@ } }, "devDependencies": { - "@backstage/cli": "^0.4.6", + "@backstage/cli": "^0.4.7", "@backstage/test-utils": "^0.1.5", "@types/archiver": "^5.1.0", "@types/compression": "^1.7.0", diff --git a/packages/backend-common/src/errors.ts b/packages/backend-common/src/errors.ts index b68dc8e3f0..39703127e6 100644 --- a/packages/backend-common/src/errors.ts +++ b/packages/backend-common/src/errors.ts @@ -75,3 +75,8 @@ export class NotFoundError extends CustomErrorBase {} * resource. */ export class ConflictError extends CustomErrorBase {} + +/** + * The requested resource has not changed since last request. + */ +export class NotModifiedError extends CustomErrorBase {} diff --git a/packages/backend-common/src/middleware/errorHandler.test.ts b/packages/backend-common/src/middleware/errorHandler.test.ts index a7a3d64bd1..6d22c2f175 100644 --- a/packages/backend-common/src/middleware/errorHandler.test.ts +++ b/packages/backend-common/src/middleware/errorHandler.test.ts @@ -72,6 +72,9 @@ describe('errorHandler', () => { it('handles well-known error classes', async () => { const app = express(); + app.use('/NotModifiedError', () => { + throw new errors.NotModifiedError(); + }); app.use('/InputError', () => { throw new errors.InputError(); }); @@ -90,6 +93,7 @@ describe('errorHandler', () => { app.use(errorHandler()); const r = request(app); + expect((await r.get('/NotModifiedError')).status).toBe(304); expect((await r.get('/InputError')).status).toBe(400); expect((await r.get('/AuthenticationError')).status).toBe(401); expect((await r.get('/NotAllowedError')).status).toBe(403); diff --git a/packages/backend-common/src/middleware/errorHandler.ts b/packages/backend-common/src/middleware/errorHandler.ts index 7365ce8b93..a08849813d 100644 --- a/packages/backend-common/src/middleware/errorHandler.ts +++ b/packages/backend-common/src/middleware/errorHandler.ts @@ -101,6 +101,8 @@ function getStatusCode(error: Error): number { // Handle well-known error types switch (error.name) { + case errors.NotModifiedError.name: + return 304; case errors.InputError.name: return 400; case errors.AuthenticationError.name: diff --git a/packages/backend-common/src/reading/AzureUrlReader.test.ts b/packages/backend-common/src/reading/AzureUrlReader.test.ts index 616cbaaadc..20f8feba42 100644 --- a/packages/backend-common/src/reading/AzureUrlReader.test.ts +++ b/packages/backend-common/src/reading/AzureUrlReader.test.ts @@ -23,6 +23,7 @@ import { getVoidLogger } from '../logging'; import { AzureUrlReader } from './AzureUrlReader'; import { msw } from '@backstage/test-utils'; import { ReadTreeResponseFactory } from './tree'; +import { NotModifiedError } from '../errors'; const logger = getVoidLogger(); @@ -139,7 +140,12 @@ describe('AzureUrlReader', () => { describe('readTree', () => { const repoBuffer = fs.readFileSync( - path.resolve('src', 'reading', '__fixtures__', 'repo.zip'), + path.resolve('src', 'reading', '__fixtures__', 'mock-main.zip'), + ); + + const processor = new AzureUrlReader( + { host: 'dev.azure.com' }, + { treeResponseFactory }, ); beforeEach(() => { @@ -153,24 +159,70 @@ describe('AzureUrlReader', () => { ctx.body(repoBuffer), ), ), + rest.get( + // https://docs.microsoft.com/en-us/rest/api/azure/devops/git/commits/get%20commits?view=azure-devops-rest-6.0#on-a-branch + 'https://dev.azure.com/organization/project/_apis/git/repositories/repository/commits', + (_, res, ctx) => + res( + ctx.status(200), + ctx.json({ + count: 2, + value: [ + { + commitId: '123abc2', + comment: 'second commit', + }, + { + commitId: '123abc1', + comment: 'first commit', + }, + ], + }), + ), + ), ); }); it('returns the wanted files from an archive', async () => { - const processor = new AzureUrlReader( - { host: 'dev.azure.com' }, - { treeResponseFactory }, - ); - const response = await processor.readTree( 'https://dev.azure.com/organization/project/_git/repository', ); + expect(response.etag).toBe('123abc2'); + const files = await response.files(); expect(files.length).toBe(2); - const mkDocsFile = await files[1].content(); - const indexMarkdownFile = await files[0].content(); + const mkDocsFile = await files[0].content(); + const indexMarkdownFile = await files[1].content(); + + expect(mkDocsFile.toString()).toBe('site_name: Test\n'); + expect(indexMarkdownFile.toString()).toBe('# Test\n'); + }); + + it('throws a NotModifiedError when given a etag in options', async () => { + const fnAzure = async () => { + await processor.readTree( + 'https://dev.azure.com/organization/project/_git/repository', + { etag: '123abc2' }, + ); + }; + + await expect(fnAzure).rejects.toThrow(NotModifiedError); + }); + + it('should not throw a NotModifiedError when given an outdated etag in options', async () => { + const response = await processor.readTree( + 'https://dev.azure.com/organization/project/_git/repository', + { etag: 'outdated123abc' }, + ); + + expect(response.etag).toBe('123abc2'); + const files = await response.files(); + + expect(files.length).toBe(2); + const mkDocsFile = await files[0].content(); + const indexMarkdownFile = await files[1].content(); expect(mkDocsFile.toString()).toBe('site_name: Test\n'); expect(indexMarkdownFile.toString()).toBe('# Test\n'); diff --git a/packages/backend-common/src/reading/AzureUrlReader.ts b/packages/backend-common/src/reading/AzureUrlReader.ts index ea716a1d4a..578db2ac92 100644 --- a/packages/backend-common/src/reading/AzureUrlReader.ts +++ b/packages/backend-common/src/reading/AzureUrlReader.ts @@ -20,10 +20,11 @@ import { getAzureFileFetchUrl, getAzureDownloadUrl, getAzureRequestOptions, + getAzureCommitsUrl, } from '@backstage/integration'; import fetch from 'cross-fetch'; import { Readable } from 'stream'; -import { NotFoundError } from '../errors'; +import { NotFoundError, NotModifiedError } from '../errors'; import { ReaderFactory, ReadTreeOptions, @@ -75,20 +76,42 @@ export class AzureUrlReader implements UrlReader { url: string, options?: ReadTreeOptions, ): Promise { - const response = await fetch( - getAzureDownloadUrl(url), - getAzureRequestOptions(this.options, { Accept: 'application/zip' }), + // TODO: Support filepath based reading tree feature like other providers + + // Get latest commit SHA + + const commitsAzureResponse = await fetch( + getAzureCommitsUrl(url), + getAzureRequestOptions(this.options), ); - if (!response.ok) { - const message = `Failed to read tree from ${url}, ${response.status} ${response.statusText}`; - if (response.status === 404) { + if (!commitsAzureResponse.ok) { + const message = `Failed to read tree from ${url}, ${commitsAzureResponse.status} ${commitsAzureResponse.statusText}`; + if (commitsAzureResponse.status === 404) { throw new NotFoundError(message); } throw new Error(message); } - return this.deps.treeResponseFactory.fromZipArchive({ - stream: (response.body as unknown) as Readable, + const commitSha = (await commitsAzureResponse.json()).value[0].commitId; + if (options?.etag && options.etag === commitSha) { + throw new NotModifiedError(); + } + + const archiveAzureResponse = await fetch( + getAzureDownloadUrl(url), + getAzureRequestOptions(this.options, { Accept: 'application/zip' }), + ); + if (!archiveAzureResponse.ok) { + const message = `Failed to read tree from ${url}, ${archiveAzureResponse.status} ${archiveAzureResponse.statusText}`; + if (archiveAzureResponse.status === 404) { + throw new NotFoundError(message); + } + throw new Error(message); + } + + return await this.deps.treeResponseFactory.fromZipArchive({ + stream: (archiveAzureResponse.body as unknown) as Readable, + etag: commitSha, filter: options?.filter, }); } diff --git a/packages/backend-common/src/reading/BitbucketUrlReader.test.ts b/packages/backend-common/src/reading/BitbucketUrlReader.test.ts index e327770114..9661368b5e 100644 --- a/packages/backend-common/src/reading/BitbucketUrlReader.test.ts +++ b/packages/backend-common/src/reading/BitbucketUrlReader.test.ts @@ -20,6 +20,7 @@ import fs from 'fs'; import { rest } from 'msw'; import { setupServer } from 'msw/node'; import path from 'path'; +import { NotModifiedError } from '../errors'; import { BitbucketUrlReader } from './BitbucketUrlReader'; import { ReadTreeResponseFactory } from './tree'; @@ -27,15 +28,24 @@ const treeResponseFactory = ReadTreeResponseFactory.create({ config: new ConfigReader({}), }); +const bitbucketProcessor = new BitbucketUrlReader( + { host: 'bitbucket.org', apiBaseUrl: 'https://api.bitbucket.org/2.0' }, + { treeResponseFactory }, +); + +const hostedBitbucketProcessor = new BitbucketUrlReader( + { + host: 'bitbucket.mycompany.net', + apiBaseUrl: 'https://api.bitbucket.mycompany.net/rest/api/1.0', + }, + { treeResponseFactory }, +); + describe('BitbucketUrlReader', () => { describe('implementation', () => { it('rejects unknown targets', async () => { - const processor = new BitbucketUrlReader( - { host: 'bitbucket.org', apiBaseUrl: 'https://api.bitbucket.org/2.0' }, - { treeResponseFactory }, - ); await expect( - processor.read('https://not.bitbucket.com/apa'), + bitbucketProcessor.read('https://not.bitbucket.com/apa'), ).rejects.toThrow( 'Incorrect URL: https://not.bitbucket.com/apa, Error: Invalid Bitbucket URL or file path', ); @@ -55,14 +65,40 @@ describe('BitbucketUrlReader', () => { ), ); - it('returns the wanted files from an archive', async () => { + const privateBitbucketRepoBuffer = fs.readFileSync( + path.resolve( + 'src', + 'reading', + '__fixtures__', + 'bitbucket-server-repo.zip', + ), + ); + + beforeEach(() => { worker.use( + rest.get( + 'https://api.bitbucket.org/2.0/repositories/backstage/mock', + (_, res, ctx) => + res( + ctx.status(200), + ctx.json({ + mainbranch: { + type: 'branch', + name: 'master', + }, + }), + ), + ), rest.get( 'https://bitbucket.org/backstage/mock/get/master.zip', (_, res, ctx) => res( ctx.status(200), ctx.set('Content-Type', 'application/zip'), + ctx.set( + 'content-disposition', + 'attachment; filename=backstage-mock-12ab34cd56ef.zip', + ), ctx.body(repoBuffer), ), ), @@ -76,17 +112,39 @@ describe('BitbucketUrlReader', () => { }), ), ), + rest.get( + 'https://api.bitbucket.mycompany.net/rest/api/1.0/projects/backstage/repos/mock/archive?format=zip&prefix=mock&path=docs', + (_, res, ctx) => + res( + ctx.status(200), + ctx.set('Content-Type', 'application/zip'), + ctx.set( + 'content-disposition', + 'attachment; filename=backstage-mock.zip', + ), + ctx.body(privateBitbucketRepoBuffer), + ), + ), + rest.get( + 'https://api.bitbucket.mycompany.net/rest/api/1.0/repositories/backstage/mock/commits/some-branch', + (_, res, ctx) => + res( + ctx.status(200), + ctx.json({ + values: [{ hash: '12ab34cd56ef78gh90ij12kl34mn56op78qr90st' }], + }), + ), + ), ); + }); - const processor = new BitbucketUrlReader( - { host: 'bitbucket.org', apiBaseUrl: 'https://api.bitbucket.org/2.0' }, - { treeResponseFactory }, - ); - - const response = await processor.readTree( + it('returns the wanted files from an archive', async () => { + const response = await bitbucketProcessor.readTree( 'https://bitbucket.org/backstage/mock/src/master', ); + expect(response.etag).toBe('12ab34cd56ef'); + const files = await response.files(); expect(files.length).toBe(2); @@ -98,38 +156,12 @@ describe('BitbucketUrlReader', () => { }); it('uses private bitbucket host', async () => { - const privateBitbucketRepoBuffer = fs.readFileSync( - path.resolve( - 'src', - 'reading', - '__fixtures__', - 'bitbucket-server-repo.zip', - ), - ); - worker.use( - rest.get( - 'https://api.bitbucket.mycompany.net/rest/api/1.0/projects/backstage/repos/mock/archive?format=zip&prefix=mock&path=docs', - (_, res, ctx) => - res( - ctx.status(200), - ctx.set('Content-Type', 'application/zip'), - ctx.body(privateBitbucketRepoBuffer), - ), - ), - ); - - const processor = new BitbucketUrlReader( - { - host: 'bitbucket.mycompany.net', - apiBaseUrl: 'https://api.bitbucket.mycompany.net/rest/api/1.0', - }, - { treeResponseFactory }, - ); - - const response = await processor.readTree( + const response = await hostedBitbucketProcessor.readTree( 'https://bitbucket.mycompany.net/projects/backstage/repos/mock/browse/docs?at=some-branch', ); + expect(response.etag).toBe('12ab34cd56ef'); + const files = await response.files(); expect(files.length).toBe(1); @@ -139,37 +171,12 @@ describe('BitbucketUrlReader', () => { }); it('returns the wanted files from an archive with a subpath', async () => { - worker.use( - rest.get( - 'https://bitbucket.org/backstage/mock/get/master.zip', - (_, res, ctx) => - res( - ctx.status(200), - ctx.set('Content-Type', 'application/zip'), - ctx.body(repoBuffer), - ), - ), - rest.get( - 'https://api.bitbucket.org/2.0/repositories/backstage/mock/commits/master', - (_, res, ctx) => - res( - ctx.status(200), - ctx.json({ - values: [{ hash: '12ab34cd56ef78gh90ij12kl34mn56op78qr90st' }], - }), - ), - ), - ); - - const processor = new BitbucketUrlReader( - { host: 'bitbucket.org', apiBaseUrl: 'https://api.bitbucket.org/2.0' }, - { treeResponseFactory }, - ); - - const response = await processor.readTree( + const response = await bitbucketProcessor.readTree( 'https://bitbucket.org/backstage/mock/src/master/docs', ); + expect(response.etag).toBe('12ab34cd56ef'); + const files = await response.files(); expect(files.length).toBe(1); @@ -177,5 +184,25 @@ describe('BitbucketUrlReader', () => { expect(indexMarkdownFile.toString()).toBe('# Test\n'); }); + + it('throws a NotModifiedError when given a etag in options', async () => { + const fnBitbucket = async () => { + await bitbucketProcessor.readTree( + 'https://bitbucket.org/backstage/mock', + { etag: '12ab34cd56ef' }, + ); + }; + + await expect(fnBitbucket).rejects.toThrow(NotModifiedError); + }); + + it('should not throw a NotModifiedError when given an outdated etag in options', async () => { + const response = await bitbucketProcessor.readTree( + 'https://bitbucket.org/backstage/mock', + { etag: 'outdatedetag123abc' }, + ); + + expect(response.etag).toBe('12ab34cd56ef'); + }); }); }); diff --git a/packages/backend-common/src/reading/BitbucketUrlReader.ts b/packages/backend-common/src/reading/BitbucketUrlReader.ts index 4578436521..e9727e04cf 100644 --- a/packages/backend-common/src/reading/BitbucketUrlReader.ts +++ b/packages/backend-common/src/reading/BitbucketUrlReader.ts @@ -25,7 +25,7 @@ import { import fetch from 'cross-fetch'; import parseGitUrl from 'git-url-parse'; import { Readable } from 'stream'; -import { NotFoundError } from '../errors'; +import { NotFoundError, NotModifiedError } from '../errors'; import { ReadTreeResponseFactory } from './tree'; import { ReaderFactory, @@ -101,34 +101,52 @@ export class BitbucketUrlReader implements UrlReader { url: string, options?: ReadTreeOptions, ): Promise { - const { name: repoName, owner: project, resource, filepath } = parseGitUrl( - url, - ); + const { filepath } = parseGitUrl(url); - const isHosted = resource === 'bitbucket.org'; + const lastCommitShortHash = await this.getLastCommitShortHash(url); + if (options?.etag && options.etag === lastCommitShortHash) { + throw new NotModifiedError(); + } const downloadUrl = await getBitbucketDownloadUrl(url, this.config); - const response = await fetch( + const archiveBitbucketResponse = await fetch( downloadUrl, getBitbucketRequestOptions(this.config), ); - if (!response.ok) { - const message = `Failed to read tree from ${url}, ${response.status} ${response.statusText}`; - if (response.status === 404) { + if (!archiveBitbucketResponse.ok) { + const message = `Failed to read tree from ${url}, ${archiveBitbucketResponse.status} ${archiveBitbucketResponse.statusText}`; + if (archiveBitbucketResponse.status === 404) { throw new NotFoundError(message); } throw new Error(message); } - let folderPath = `${project}-${repoName}`; - if (isHosted) { - const lastCommitShortHash = await this.getLastCommitShortHash(url); - folderPath = `${project}-${repoName}-${lastCommitShortHash}`; + // Get the filename of archive from the header of the response + const contentDispositionHeader = archiveBitbucketResponse.headers.get( + 'content-disposition', + ) as string; + if (!contentDispositionHeader) { + throw new Error( + `Failed to read tree from ${url}. ` + + 'Bitbucket API response for downloading archive does not contain content-disposition header ', + ); + } + const fileNameRegEx = new RegExp( + /^attachment; filename=(?.*).zip$/, + ); + const archiveFileName = contentDispositionHeader.match(fileNameRegEx) + ?.groups?.fileName; + if (!archiveFileName) { + throw new Error( + `Failed to read tree from ${url}. Bitbucket API response for downloading archive has an unexpected ` + + `format of content-disposition header ${contentDispositionHeader} `, + ); } - return this.treeResponseFactory.fromZipArchive({ - stream: (response.body as unknown) as Readable, - path: `${folderPath}/${filepath}`, + return await this.treeResponseFactory.fromZipArchive({ + stream: (archiveBitbucketResponse.body as unknown) as Readable, + path: `${archiveFileName}/${filepath}`, + etag: lastCommitShortHash, filter: options?.filter, }); } @@ -142,7 +160,7 @@ export class BitbucketUrlReader implements UrlReader { return `bitbucket{host=${host},authed=${authed}}`; } - private async getLastCommitShortHash(url: string): Promise { + private async getLastCommitShortHash(url: string): Promise { const { name: repoName, owner: project, ref } = parseGitUrl(url); let branch = ref; diff --git a/packages/backend-common/src/reading/GithubUrlReader.test.ts b/packages/backend-common/src/reading/GithubUrlReader.test.ts index 1a0f27d28f..080e8b1d5b 100644 --- a/packages/backend-common/src/reading/GithubUrlReader.test.ts +++ b/packages/backend-common/src/reading/GithubUrlReader.test.ts @@ -21,6 +21,7 @@ import fs from 'fs'; import { rest } from 'msw'; import { setupServer } from 'msw/node'; import path from 'path'; +import { NotFoundError, NotModifiedError } from '../errors'; import { GithubUrlReader } from './GithubUrlReader'; import { ReadTreeResponseFactory } from './tree'; @@ -28,11 +29,27 @@ const treeResponseFactory = ReadTreeResponseFactory.create({ config: new ConfigReader({}), }); -describe('GithubUrlReader', () => { - const mockCredentialsProvider = ({ - getCredentials: jest.fn().mockResolvedValue({ headers: {} }), - } as unknown) as GithubCredentialsProvider; +const mockCredentialsProvider = ({ + getCredentials: jest.fn().mockResolvedValue({ headers: {} }), +} as unknown) as GithubCredentialsProvider; +const githubProcessor = new GithubUrlReader( + { + host: 'github.com', + apiBaseUrl: 'https://api.github.com', + }, + { treeResponseFactory, credentialsProvider: mockCredentialsProvider }, +); + +const gheProcessor = new GithubUrlReader( + { + host: 'ghe.github.com', + apiBaseUrl: 'https://ghe.github.com/api/v3', + }, + { treeResponseFactory, credentialsProvider: mockCredentialsProvider }, +); + +describe('GithubUrlReader', () => { const worker = setupServer(); msw.setupDefaultHandlers(worker); @@ -43,15 +60,8 @@ describe('GithubUrlReader', () => { describe('implementation', () => { it('rejects unknown targets', async () => { - const processor = new GithubUrlReader( - { - host: 'github.com', - apiBaseUrl: 'https://api.github.com', - }, - { treeResponseFactory, credentialsProvider: mockCredentialsProvider }, - ); await expect( - processor.read('https://not.github.com/apa'), + githubProcessor.read('https://not.github.com/apa'), ).rejects.toThrow( 'Incorrect URL: https://not.github.com/apa, Error: Invalid GitHub URL or file path', ); @@ -73,7 +83,7 @@ describe('GithubUrlReader', () => { worker.use( rest.get( - 'https://api.github.com/repos/backstage/mock/tree/contents/?ref=repo', + 'https://api.github.com/repos/backstage/mock/tree/contents/?ref=main', (req, res, ctx) => { expect(req.headers.get('authorization')).toBe( mockHeaders.Authorization, @@ -90,50 +100,122 @@ describe('GithubUrlReader', () => { ), ); - const processor = new GithubUrlReader( - { - host: 'ghe.github.com', - apiBaseUrl: 'https://api.github.com', - }, - { treeResponseFactory, credentialsProvider: mockCredentialsProvider }, - ); - await processor.read( - 'https://ghe.github.com/backstage/mock/tree/blob/repo', + await githubProcessor.read( + 'https://github.com/backstage/mock/tree/blob/main', ); }); }); describe('readTree', () => { const repoBuffer = fs.readFileSync( - path.resolve('src', 'reading', '__fixtures__', 'repo.tar.gz'), + path.resolve( + 'src', + 'reading', + '__fixtures__', + 'backstage-mock-etag123.tar.gz', + ), ); + const reposGithubApiResponse = { + id: '123', + full_name: 'backstage/mock', + default_branch: 'main', + branches_url: + 'https://api.github.com/repos/backstage/mock/branches{/branch}', + archive_url: + 'https://api.github.com/repos/backstage/mock/{archive_format}{/ref}', + }; + + const reposGheApiResponse = { + ...reposGithubApiResponse, + branches_url: + 'https://ghe.github.com/api/v3/repos/backstage/mock/branches{/branch}', + archive_url: + 'https://ghe.github.com/api/v3/repos/backstage/mock/{archive_format}{/ref}', + }; + + const branchesApiResponse = { + name: 'main', + commit: { + sha: 'etag123abc', + }, + }; + beforeEach(() => { worker.use( + rest.get('https://api.github.com/repos/backstage/mock', (_, res, ctx) => + res( + ctx.status(200), + ctx.set('Content-Type', 'application/json'), + ctx.json(reposGithubApiResponse), + ), + ), rest.get( - 'https://github.com/backstage/mock/archive/repo.tar.gz', + 'https://api.github.com/repos/backstage/mock/branches/main', + (_, res, ctx) => + res( + ctx.status(200), + ctx.set('Content-Type', 'application/json'), + ctx.json(branchesApiResponse), + ), + ), + rest.get( + 'https://api.github.com/repos/backstage/mock/tarball/etag123abc', (_, res, ctx) => res( ctx.status(200), ctx.set('Content-Type', 'application/x-gzip'), + ctx.set( + 'content-disposition', + 'attachment; filename=backstage-mock-etag123.tar.gz', + ), ctx.body(repoBuffer), ), ), + rest.get( + 'https://api.github.com/repos/backstage/mock/branches/branchDoesNotExist', + (_, res, ctx) => res(ctx.status(404)), + ), + rest.get( + 'https://ghe.github.com/api/v3/repos/backstage/mock/tarball/etag123abc', + (_, res, ctx) => + res( + ctx.status(200), + ctx.set('Content-Type', 'application/x-gzip'), + ctx.set( + 'content-disposition', + 'attachment; filename=backstage-mock-etag123.tar.gz', + ), + ctx.body(repoBuffer), + ), + ), + rest.get( + 'https://ghe.github.com/api/v3/repos/backstage/mock', + (_, res, ctx) => + res( + ctx.status(200), + ctx.set('Content-Type', 'application/json'), + ctx.json(reposGheApiResponse), + ), + ), + rest.get( + 'https://ghe.github.com/api/v3/repos/backstage/mock/branches/main', + (_, res, ctx) => + res( + ctx.status(200), + ctx.set('Content-Type', 'application/json'), + ctx.json(branchesApiResponse), + ), + ), ); }); it('returns the wanted files from an archive', async () => { - const processor = new GithubUrlReader( - { - host: 'github.com', - apiBaseUrl: 'https://api.github.com', - }, - { treeResponseFactory, credentialsProvider: mockCredentialsProvider }, + const response = await githubProcessor.readTree( + 'https://github.com/backstage/mock/tree/main', ); - const response = await processor.readTree( - 'https://github.com/backstage/mock/tree/repo', - ); + expect(response.etag).toBe('etag123abc'); const files = await response.files(); @@ -145,40 +227,6 @@ describe('GithubUrlReader', () => { expect(indexMarkdownFile.toString()).toBe('# Test\n'); }); - it('includes the subdomain in the github url', async () => { - worker.resetHandlers(); - worker.use( - rest.get( - 'https://ghe.github.com/backstage/mock/archive/repo.tar.gz', - (_, res, ctx) => - res( - ctx.status(200), - ctx.set('Content-Type', 'application/x-gzip'), - ctx.body(repoBuffer), - ), - ), - ); - - const processor = new GithubUrlReader( - { - host: 'ghe.github.com', - apiBaseUrl: 'https://api.github.com', - }, - { treeResponseFactory, credentialsProvider: mockCredentialsProvider }, - ); - - const response = await processor.readTree( - 'https://ghe.github.com/backstage/mock/tree/repo/docs', - ); - - const files = await response.files(); - - expect(files.length).toBe(1); - const indexMarkdownFile = await files[0].content(); - - expect(indexMarkdownFile.toString()).toBe('# Test\n'); - }); - it('should use the headers from the credentials provider to the fetch request', async () => { expect.assertions(2); @@ -193,7 +241,7 @@ describe('GithubUrlReader', () => { worker.use( rest.get( - 'https://ghe.github.com/backstage/mock/archive/repo.tar.gz', + 'https://ghe.github.com/api/v3/repos/backstage/mock/tarball/etag123abc', (req, res, ctx) => { expect(req.headers.get('authorization')).toBe( mockHeaders.Authorization, @@ -204,52 +252,24 @@ describe('GithubUrlReader', () => { return res( ctx.status(200), ctx.set('Content-Type', 'application/x-gzip'), + ctx.set( + 'content-disposition', + 'attachment; filename=backstage-mock-etag123.tar.gz', + ), ctx.body(repoBuffer), ); }, ), ); - const processor = new GithubUrlReader( - { - host: 'ghe.github.com', - apiBaseUrl: 'https://api.github.com', - }, - { treeResponseFactory, credentialsProvider: mockCredentialsProvider }, - ); - - await processor.readTree( - 'https://ghe.github.com/backstage/mock/tree/repo/docs', + await gheProcessor.readTree( + 'https://ghe.github.com/backstage/mock/tree/main', ); }); - it('must specify a branch', async () => { - const processor = new GithubUrlReader( - { - host: 'github.com', - apiBaseUrl: 'https://api.github.com', - }, - { treeResponseFactory, credentialsProvider: mockCredentialsProvider }, - ); - - await expect( - processor.readTree('https://github.com/backstage/mock'), - ).rejects.toThrow( - 'GitHub URL must contain branch to be able to fetch tree', - ); - }); - - it('returns the wanted files from an archive with a subpath', async () => { - const processor = new GithubUrlReader( - { - host: 'github.com', - apiBaseUrl: 'https://api.github.com', - }, - { treeResponseFactory, credentialsProvider: mockCredentialsProvider }, - ); - - const response = await processor.readTree( - 'https://github.com/backstage/mock/tree/repo/docs', + it('includes the subdomain in the github url', async () => { + const response = await gheProcessor.readTree( + 'https://ghe.github.com/backstage/mock/tree/main/docs', ); const files = await response.files(); @@ -259,5 +279,64 @@ describe('GithubUrlReader', () => { expect(indexMarkdownFile.toString()).toBe('# Test\n'); }); + + it('returns the wanted files from an archive with a subpath', async () => { + const response = await githubProcessor.readTree( + 'https://github.com/backstage/mock/tree/main/docs', + ); + + const files = await response.files(); + + expect(files.length).toBe(1); + const indexMarkdownFile = await files[0].content(); + + expect(indexMarkdownFile.toString()).toBe('# Test\n'); + }); + + it('throws a NotModifiedError when given a etag in options', async () => { + const fnGithub = async () => { + await githubProcessor.readTree('https://github.com/backstage/mock', { + etag: 'etag123abc', + }); + }; + + const fnGhe = async () => { + await gheProcessor.readTree( + 'https://ghe.github.com/backstage/mock/tree/main/docs', + { + etag: 'etag123abc', + }, + ); + }; + + await expect(fnGithub).rejects.toThrow(NotModifiedError); + await expect(fnGhe).rejects.toThrow(NotModifiedError); + }); + + it('should not throw error when given an outdated etag in options', async () => { + const response = await githubProcessor.readTree( + 'https://github.com/backstage/mock/tree/main', + { + etag: 'outdatedetag123abc', + }, + ); + expect((await response.files()).length).toBe(2); + }); + + it('should detect the default branch', async () => { + const response = await githubProcessor.readTree( + 'https://github.com/backstage/mock', + ); + expect((await response.files()).length).toBe(2); + }); + + it('should throw error on missing branch', async () => { + const fnGithub = async () => { + await githubProcessor.readTree( + 'https://github.com/backstage/mock/tree/branchDoesNotExist', + ); + }; + await expect(fnGithub).rejects.toThrow(NotFoundError); + }); }); }); diff --git a/packages/backend-common/src/reading/GithubUrlReader.ts b/packages/backend-common/src/reading/GithubUrlReader.ts index 45ce8336e6..6c7cefe2ef 100644 --- a/packages/backend-common/src/reading/GithubUrlReader.ts +++ b/packages/backend-common/src/reading/GithubUrlReader.ts @@ -23,7 +23,7 @@ import { import fetch from 'cross-fetch'; import parseGitUrl from 'git-url-parse'; import { Readable } from 'stream'; -import { InputError, NotFoundError } from '../errors'; +import { NotFoundError, NotModifiedError } from '../errors'; import { ReadTreeResponseFactory } from './tree'; import { ReaderFactory, @@ -98,51 +98,106 @@ export class GithubUrlReader implements UrlReader { url: string, options?: ReadTreeOptions, ): Promise { - const { - name: repoName, - ref, - protocol, - resource, - full_name, - filepath, - } = parseGitUrl(url); - - if (!ref) { - // TODO(Rugvip): We should add support for defaulting to the default branch - throw new InputError( - 'GitHub URL must contain branch to be able to fetch tree', - ); - } + const { ref, filepath, full_name } = parseGitUrl(url); + // Caveat: The ref will totally be incorrect if the branch name includes a / + // Thus, readTree can not work on url containing branch name that has a / const { headers } = await this.deps.credentialsProvider.getCredentials({ url, }); - // TODO(Rugvip): use API to fetch URL instead - const response = await fetch( - new URL( - `${protocol}://${resource}/${full_name}/archive/${ref}.tar.gz`, - ).toString(), + + // Get GitHub API urls for the repository + const repoGitHubResponse = await fetch( + new URL(`${this.config.apiBaseUrl}/repos/${full_name}`).toString(), { - headers: { - ...headers, - }, + headers, }, ); - if (!response.ok) { - const message = `Failed to read tree from ${url}, ${response.status} ${response.statusText}`; - if (response.status === 404) { + if (!repoGitHubResponse.ok) { + const message = `Failed to read tree (repository) from ${url}, ${repoGitHubResponse.status} ${repoGitHubResponse.statusText}`; + if (repoGitHubResponse.status === 404) { throw new NotFoundError(message); } throw new Error(message); } - const path = `${repoName}-${ref}/${filepath}`; + const repoResponseJson = await repoGitHubResponse.json(); - return this.deps.treeResponseFactory.fromTarArchive({ + // ref is an empty string if no branch is set in provided url to readTree. + // Use GitHub API to get the default branch of the repository. + const branch = ref || repoResponseJson.default_branch; + const branchesApiUrl = repoResponseJson.branches_url; + const archiveApiUrl = repoResponseJson.archive_url; + + // Fetch the latest commit in the provided or default branch to compare against + // the provided sha. + const branchGitHubResponse = await fetch( + // branchesApiUrl looks like "https://api.github.com/repos/owner/repo/branches{/branch}" + branchesApiUrl.replace('{/branch}', `/${branch}`), + { + headers, + }, + ); + if (!branchGitHubResponse.ok) { + const message = `Failed to read tree (branch) from ${url}, ${branchGitHubResponse.status} ${branchGitHubResponse.statusText}`; + if (branchGitHubResponse.status === 404) { + throw new NotFoundError(message); + } + throw new Error(message); + } + const commitSha = (await branchGitHubResponse.json()).commit.sha; + + if (options?.etag && options.etag === commitSha) { + throw new NotModifiedError(); + } + + const archive = await fetch( + // archiveApiUrl looks like "https://api.github.com/repos/owner/repo/{archive_format}{/ref}" + archiveApiUrl + .replace('{archive_format}', 'tarball') + .replace('{/ref}', `/${commitSha}`), + { headers }, + ); + if (!archive.ok) { + const message = `Failed to read tree (archive) from ${url}, ${archive.status} ${archive.statusText}`; + if (archive.status === 404) { + throw new NotFoundError(message); + } + throw new Error(message); + } + + // Get the filename of archive from the header of the response + const contentDispositionHeader = archive.headers.get( + 'content-disposition', + ) as string; + if (!contentDispositionHeader) { + throw new Error( + `Failed to read tree from ${url}. ` + + 'GitHub API response for downloading archive does not contain content-disposition header ', + ); + } + const fileNameRegEx = new RegExp( + /^attachment; filename=(?.*).tar.gz$/, + ); + const archiveFileName = contentDispositionHeader.match(fileNameRegEx) + ?.groups?.fileName; + if (!archiveFileName) { + throw new Error( + `Failed to read tree from ${url}. GitHub API response for downloading archive has an unexpected ` + + `format of content-disposition header ${contentDispositionHeader} `, + ); + } + + // The path includes the name of the directory inside the tarball and a sub path + // if requested in readTree. + const path = `${archiveFileName}/${filepath}`; + + return await this.deps.treeResponseFactory.fromTarArchive({ // TODO(Rugvip): Underlying implementation of fetch will be node-fetch, we probably want // to stick to using that in exclusively backend code. - stream: (response.body as unknown) as Readable, + stream: (archive.body as unknown) as Readable, path, + etag: commitSha, filter: options?.filter, }); } diff --git a/packages/backend-common/src/reading/GitlabUrlReader.test.ts b/packages/backend-common/src/reading/GitlabUrlReader.test.ts index 6cc5e30dbd..c0736f769d 100644 --- a/packages/backend-common/src/reading/GitlabUrlReader.test.ts +++ b/packages/backend-common/src/reading/GitlabUrlReader.test.ts @@ -23,6 +23,7 @@ import path from 'path'; import { getVoidLogger } from '../logging'; import { GitlabUrlReader } from './GitlabUrlReader'; import { ReadTreeResponseFactory } from './tree'; +import { NotModifiedError, NotFoundError } from '../errors'; const logger = getVoidLogger(); @@ -30,6 +31,22 @@ const treeResponseFactory = ReadTreeResponseFactory.create({ config: new ConfigReader({}), }); +const gitlabProcessor = new GitlabUrlReader( + { + host: 'gitlab.com', + apiBaseUrl: 'https://gitlab.com/api/v4', + }, + { treeResponseFactory }, +); + +const hostedGitlabProcessor = new GitlabUrlReader( + { + host: 'gitlab.mycompany.com', + apiBaseUrl: 'https://gitlab.mycompany.com/api/v4', + }, + { treeResponseFactory }, +); + describe('GitlabUrlReader', () => { const worker = setupServer(); msw.setupDefaultHandlers(worker); @@ -136,39 +153,102 @@ describe('GitlabUrlReader', () => { }); describe('readTree', () => { - const repoBuffer = fs.readFileSync( - path.resolve('src', 'reading', '__fixtures__', 'repo.zip'), + const archiveBuffer = fs.readFileSync( + path.resolve('src', 'reading', '__fixtures__', 'gitlab-archive.zip'), ); + const projectGitlabApiResponse = { + id: 11111111, + default_branch: 'main', + }; + + const branchGitlabApiResponse = { + commit: { + id: 'sha123abc', + }, + }; + beforeEach(() => { worker.use( rest.get( - 'https://gitlab.com/backstage/mock/-/archive/repo/mock-repo.zip', + 'https://gitlab.com/api/v4/projects/backstage%2Fmock/repository/archive.zip?sha=main', (_, res, ctx) => res( ctx.status(200), ctx.set('Content-Type', 'application/zip'), - ctx.body(repoBuffer), + ctx.set( + 'content-disposition', + 'attachment; filename="mock-main-sha123abc.zip"', + ), + ctx.body(archiveBuffer), + ), + ), + rest.get( + 'https://gitlab.com/api/v4/projects/backstage%2Fmock', + (_, res, ctx) => + res( + ctx.status(200), + ctx.set('Content-Type', 'application/json'), + ctx.json(projectGitlabApiResponse), + ), + ), + rest.get( + 'https://gitlab.com/api/v4/projects/backstage%2Fmock/repository/branches/main', + (_, res, ctx) => + res( + ctx.status(200), + ctx.set('Content-Type', 'application/json'), + ctx.json(branchGitlabApiResponse), + ), + ), + rest.get( + 'https://gitlab.com/api/v4/projects/backstage%2Fmock/repository/branches/branchDoesNotExist', + (_, res, ctx) => res(ctx.status(404)), + ), + rest.get( + 'https://gitlab.mycompany.com/api/v4/projects/backstage%2Fmock', + (_, res, ctx) => + res( + ctx.status(200), + ctx.set('Content-Type', 'application/json'), + ctx.json(projectGitlabApiResponse), + ), + ), + rest.get( + 'https://gitlab.mycompany.com/api/v4/projects/backstage%2Fmock/repository/branches/main', + (_, res, ctx) => + res( + ctx.status(200), + ctx.set('Content-Type', 'application/json'), + ctx.json(branchGitlabApiResponse), + ), + ), + rest.get( + 'https://gitlab.mycompany.com/api/v4/projects/backstage%2Fmock/repository/archive.zip?sha=main', + (_, res, ctx) => + res( + ctx.status(200), + ctx.set('Content-Type', 'application/zip'), + ctx.set( + 'content-disposition', + 'attachment; filename="mock-main-sha123abc.zip"', + ), + ctx.body(archiveBuffer), ), ), ); }); it('returns the wanted files from an archive', async () => { - const processor = new GitlabUrlReader( - { host: 'gitlab.com' }, - { treeResponseFactory }, - ); - - const response = await processor.readTree( - 'https://gitlab.com/backstage/mock/tree/repo', + const response = await gitlabProcessor.readTree( + 'https://gitlab.com/backstage/mock/tree/main', ); const files = await response.files(); expect(files.length).toBe(2); - const indexMarkdownFile = await files[0].content(); - const mkDocsFile = await files[1].content(); + const mkDocsFile = await files[0].content(); + const indexMarkdownFile = await files[1].content(); expect(mkDocsFile.toString()).toBe('site_name: Test\n'); expect(indexMarkdownFile.toString()).toBe('# Test\n'); @@ -177,23 +257,22 @@ describe('GitlabUrlReader', () => { it('returns the wanted files from hosted gitlab', async () => { worker.use( rest.get( - 'https://git.mycompany.com/backstage/mock/-/archive/repo/mock-repo.zip', + 'https://gitlab.mycompany.com/backstage/mock/-/archive/main.zip', (_, res, ctx) => res( ctx.status(200), ctx.set('Content-Type', 'application/zip'), - ctx.body(repoBuffer), + ctx.set( + 'content-disposition', + 'attachment; filename="mock-main-sha123abc.zip"', + ), + ctx.body(archiveBuffer), ), ), ); - const processor = new GitlabUrlReader( - { host: 'git.mycompany.com' }, - { treeResponseFactory }, - ); - - const response = await processor.readTree( - 'https://git.mycompany.com/backstage/mock/tree/repo/docs', + const response = await hostedGitlabProcessor.readTree( + 'https://gitlab.mycompany.com/backstage/mock/tree/main/docs', ); const files = await response.files(); @@ -204,27 +283,9 @@ describe('GitlabUrlReader', () => { expect(indexMarkdownFile.toString()).toBe('# Test\n'); }); - it('throws an error when branch is not specified', async () => { - const processor = new GitlabUrlReader( - { host: 'gitlab.com' }, - { treeResponseFactory }, - ); - - await expect( - processor.readTree('https://gitlab.com/backstage/mock'), - ).rejects.toThrow( - 'GitLab URL must contain a branch to be able to fetch its tree', - ); - }); - it('returns the wanted files from an archive with a subpath', async () => { - const processor = new GitlabUrlReader( - { host: 'gitlab.com' }, - { treeResponseFactory }, - ); - - const response = await processor.readTree( - 'https://gitlab.com/backstage/mock/tree/repo/docs', + const response = await gitlabProcessor.readTree( + 'https://gitlab.com/backstage/mock/tree/main/docs', ); const files = await response.files(); @@ -234,5 +295,51 @@ describe('GitlabUrlReader', () => { expect(indexMarkdownFile.toString()).toBe('# Test\n'); }); + + it('throws a NotModifiedError when given a etag in options', async () => { + const fnGitlab = async () => { + await gitlabProcessor.readTree('https://gitlab.com/backstage/mock', { + etag: 'sha123abc', + }); + }; + + const fnHostedGitlab = async () => { + await hostedGitlabProcessor.readTree( + 'https://gitlab.mycompany.com/backstage/mock', + { + etag: 'sha123abc', + }, + ); + }; + + await expect(fnGitlab).rejects.toThrow(NotModifiedError); + await expect(fnHostedGitlab).rejects.toThrow(NotModifiedError); + }); + + it('should not throw error when given an outdated etag in options', async () => { + const response = await gitlabProcessor.readTree( + 'https://gitlab.com/backstage/mock/tree/main', + { + etag: 'outdatedsha123abc', + }, + ); + expect((await response.files()).length).toBe(2); + }); + + it('should detect the default branch', async () => { + const response = await gitlabProcessor.readTree( + 'https://gitlab.com/backstage/mock', + ); + expect((await response.files()).length).toBe(2); + }); + + it('should throw error on missing branch', async () => { + const fnGithub = async () => { + await gitlabProcessor.readTree( + 'https://gitlab.com/backstage/mock/tree/branchDoesNotExist', + ); + }; + await expect(fnGithub).rejects.toThrow(NotFoundError); + }); }); }); diff --git a/packages/backend-common/src/reading/GitlabUrlReader.ts b/packages/backend-common/src/reading/GitlabUrlReader.ts index 299bbc783b..654f4f9a85 100644 --- a/packages/backend-common/src/reading/GitlabUrlReader.ts +++ b/packages/backend-common/src/reading/GitlabUrlReader.ts @@ -21,7 +21,7 @@ import { readGitLabIntegrationConfigs, } from '@backstage/integration'; import fetch from 'cross-fetch'; -import { InputError, NotFoundError } from '../errors'; +import { NotFoundError, NotModifiedError } from '../errors'; import { ReadTreeResponseFactory } from './tree'; import { ReaderFactory, @@ -39,26 +39,26 @@ export class GitlabUrlReader implements UrlReader { const configs = readGitLabIntegrationConfigs( config.getOptionalConfigArray('integrations.gitlab') ?? [], ); - return configs.map(options => { - const reader = new GitlabUrlReader(options, { treeResponseFactory }); - const predicate = (url: URL) => url.host === options.host; + return configs.map(provider => { + const reader = new GitlabUrlReader(provider, { treeResponseFactory }); + const predicate = (url: URL) => url.host === provider.host; return { reader, predicate }; }); }; constructor( - private readonly options: GitLabIntegrationConfig, + private readonly config: GitLabIntegrationConfig, deps: { treeResponseFactory: ReadTreeResponseFactory }, ) { this.treeResponseFactory = deps.treeResponseFactory; } async read(url: string): Promise { - const builtUrl = await getGitLabFileFetchUrl(url, this.options); + const builtUrl = await getGitLabFileFetchUrl(url, this.config); let response: Response; try { - response = await fetch(builtUrl, getGitLabRequestOptions(this.options)); + response = await fetch(builtUrl, getGitLabRequestOptions(this.config)); } catch (e) { throw new Error(`Unable to read ${url}, ${e}`); } @@ -78,45 +78,102 @@ export class GitlabUrlReader implements UrlReader { url: string, options?: ReadTreeOptions, ): Promise { - const { - name: repoName, - ref, - protocol, - resource, - full_name, - filepath, - } = parseGitUrl(url); + const { ref, full_name, filepath } = parseGitUrl(url); - if (!ref) { - throw new InputError( - 'GitLab URL must contain a branch to be able to fetch its tree', - ); - } - - const archive = `${protocol}://${resource}/${full_name}/-/archive/${ref}/${repoName}-${ref}.zip`; - const response = await fetch( - archive, - getGitLabRequestOptions(this.options), + // Use GitLab API to get the default branch + // encodeURIComponent is required for GitLab API + // https://docs.gitlab.com/ee/api/README.html#namespaced-path-encoding + const projectGitlabResponse = await fetch( + new URL( + `${this.config.apiBaseUrl}/projects/${encodeURIComponent(full_name)}`, + ).toString(), + getGitLabRequestOptions(this.config), ); - if (!response.ok) { - const msg = `Failed to read tree from ${url}, ${response.status} ${response.statusText}`; - if (response.status === 404) { + if (!projectGitlabResponse.ok) { + const msg = `Failed to read tree from ${url}, ${projectGitlabResponse.status} ${projectGitlabResponse.statusText}`; + if (projectGitlabResponse.status === 404) { throw new NotFoundError(msg); } throw new Error(msg); } + const projectGitlabResponseJson = await projectGitlabResponse.json(); - const path = filepath ? `${repoName}-${ref}/${filepath}/` : ''; + // ref is an empty string if no branch is set in provided url to readTree. + const branch = ref || projectGitlabResponseJson.default_branch; - return this.treeResponseFactory.fromZipArchive({ - stream: (response.body as unknown) as Readable, + // Fetch the latest commit in the provided or default branch to compare against + // the provided sha. + const branchGitlabResponse = await fetch( + new URL( + `${this.config.apiBaseUrl}/projects/${encodeURIComponent( + full_name, + )}/repository/branches/${branch}`, + ).toString(), + getGitLabRequestOptions(this.config), + ); + if (!branchGitlabResponse.ok) { + const message = `Failed to read tree (branch) from ${url}, ${branchGitlabResponse.status} ${branchGitlabResponse.statusText}`; + if (branchGitlabResponse.status === 404) { + throw new NotFoundError(message); + } + throw new Error(message); + } + + const commitSha = (await branchGitlabResponse.json()).commit.id; + + if (options?.etag && options.etag === commitSha) { + throw new NotModifiedError(); + } + + // https://docs.gitlab.com/ee/api/repositories.html#get-file-archive + const archiveGitLabResponse = await fetch( + `${this.config.apiBaseUrl}/projects/${encodeURIComponent( + full_name, + )}/repository/archive.zip?sha=${branch}`, + getGitLabRequestOptions(this.config), + ); + if (!archiveGitLabResponse.ok) { + const message = `Failed to read tree (archive) from ${url}, ${archiveGitLabResponse.status} ${archiveGitLabResponse.statusText}`; + if (archiveGitLabResponse.status === 404) { + throw new NotFoundError(message); + } + throw new Error(message); + } + + // Get the filename of archive from the header of the response + const contentDispositionHeader = archiveGitLabResponse.headers.get( + 'content-disposition', + ) as string; + if (!contentDispositionHeader) { + throw new Error( + `Failed to read tree from ${url}. ` + + 'GitLab API response for downloading archive does not contain content-disposition header ', + ); + } + const fileNameRegEx = new RegExp( + /^attachment; filename="(?.*).zip"$/, + ); + const archiveFileName = contentDispositionHeader.match(fileNameRegEx) + ?.groups?.fileName; + if (!archiveFileName) { + throw new Error( + `Failed to read tree from ${url}. GitLab API response for downloading archive has an unexpected ` + + `format of content-disposition header ${contentDispositionHeader} `, + ); + } + + const path = filepath ? `${archiveFileName}/${filepath}/` : ''; + + return await this.treeResponseFactory.fromZipArchive({ + stream: (archiveGitLabResponse.body as unknown) as Readable, path, + etag: commitSha, filter: options?.filter, }); } toString() { - const { host, token } = this.options; + const { host, token } = this.config; return `gitlab{host=${host},authed=${Boolean(token)}}`; } } diff --git a/packages/backend-common/src/reading/UrlReaderPredicateMux.ts b/packages/backend-common/src/reading/UrlReaderPredicateMux.ts index ca11400e4a..3183aa0c28 100644 --- a/packages/backend-common/src/reading/UrlReaderPredicateMux.ts +++ b/packages/backend-common/src/reading/UrlReaderPredicateMux.ts @@ -45,12 +45,15 @@ export class UrlReaderPredicateMux implements UrlReader { throw new NotAllowedError(`Reading from '${url}' is not allowed`); } - readTree(url: string, options?: ReadTreeOptions): Promise { + async readTree( + url: string, + options?: ReadTreeOptions, + ): Promise { const parsed = new URL(url); for (const { predicate, reader } of this.readers) { if (predicate(parsed)) { - return reader.readTree(url, options); + return await reader.readTree(url, options); } } diff --git a/packages/backend-common/src/reading/__fixtures__/backstage-mock-etag123.tar.gz b/packages/backend-common/src/reading/__fixtures__/backstage-mock-etag123.tar.gz new file mode 100644 index 0000000000..e1ac2579de Binary files /dev/null and b/packages/backend-common/src/reading/__fixtures__/backstage-mock-etag123.tar.gz differ diff --git a/packages/backend-common/src/reading/__fixtures__/gitlab-archive.zip b/packages/backend-common/src/reading/__fixtures__/gitlab-archive.zip new file mode 100644 index 0000000000..884ec20004 Binary files /dev/null and b/packages/backend-common/src/reading/__fixtures__/gitlab-archive.zip differ diff --git a/packages/backend-common/src/reading/__fixtures__/mock-main.tar.gz b/packages/backend-common/src/reading/__fixtures__/mock-main.tar.gz new file mode 100644 index 0000000000..291690b447 Binary files /dev/null and b/packages/backend-common/src/reading/__fixtures__/mock-main.tar.gz differ diff --git a/packages/backend-common/src/reading/__fixtures__/mock-main.zip b/packages/backend-common/src/reading/__fixtures__/mock-main.zip new file mode 100644 index 0000000000..beee59d3a0 Binary files /dev/null and b/packages/backend-common/src/reading/__fixtures__/mock-main.zip differ diff --git a/packages/backend-common/src/reading/__fixtures__/repo.tar.gz b/packages/backend-common/src/reading/__fixtures__/repo.tar.gz deleted file mode 100644 index 7a8e9902a2..0000000000 Binary files a/packages/backend-common/src/reading/__fixtures__/repo.tar.gz and /dev/null differ diff --git a/packages/backend-common/src/reading/__fixtures__/repo.zip b/packages/backend-common/src/reading/__fixtures__/repo.zip deleted file mode 100644 index f66bf2d612..0000000000 Binary files a/packages/backend-common/src/reading/__fixtures__/repo.zip and /dev/null differ diff --git a/packages/backend-common/src/reading/tree/ReadTreeResponseFactory.ts b/packages/backend-common/src/reading/tree/ReadTreeResponseFactory.ts index 986a0302bc..7332154d09 100644 --- a/packages/backend-common/src/reading/tree/ReadTreeResponseFactory.ts +++ b/packages/backend-common/src/reading/tree/ReadTreeResponseFactory.ts @@ -26,6 +26,8 @@ type FromArchiveOptions = { stream: Readable; // If set, the root of the tree will be set to the given directory path. path?: string; + // etag of the blob + etag: string; // Filter passed on from the ReadTreeOptions filter?: (path: string) => boolean; }; @@ -45,6 +47,7 @@ export class ReadTreeResponseFactory { options.stream, options.path ?? '', this.workDir, + options.etag, options.filter, ); } @@ -54,6 +57,7 @@ export class ReadTreeResponseFactory { options.stream, options.path ?? '', this.workDir, + options.etag, options.filter, ); } diff --git a/packages/backend-common/src/reading/tree/TarArchiveResponse.test.ts b/packages/backend-common/src/reading/tree/TarArchiveResponse.test.ts index 1bc0d3a386..2cbfc4a89e 100644 --- a/packages/backend-common/src/reading/tree/TarArchiveResponse.test.ts +++ b/packages/backend-common/src/reading/tree/TarArchiveResponse.test.ts @@ -20,7 +20,7 @@ import { resolve as resolvePath } from 'path'; import { TarArchiveResponse } from './TarArchiveResponse'; const archiveData = fs.readFileSync( - resolvePath(__filename, '../../__fixtures__/repo.tar.gz'), + resolvePath(__filename, '../../__fixtures__/mock-main.tar.gz'), ); describe('TarArchiveResponse', () => { @@ -38,7 +38,7 @@ describe('TarArchiveResponse', () => { it('should read files', async () => { const stream = fs.createReadStream('/test-archive.tar.gz'); - const res = new TarArchiveResponse(stream, 'mock-repo/', '/tmp'); + const res = new TarArchiveResponse(stream, 'mock-main/', '/tmp', 'etag'); const files = await res.files(); expect(files).toEqual([ @@ -61,8 +61,12 @@ describe('TarArchiveResponse', () => { it('should read files with filter', async () => { const stream = fs.createReadStream('/test-archive.tar.gz'); - const res = new TarArchiveResponse(stream, 'mock-repo/', '/tmp', path => - path.endsWith('.yml'), + const res = new TarArchiveResponse( + stream, + 'mock-main/', + '/tmp', + 'etag', + path => path.endsWith('.yml'), ); const files = await res.files(); @@ -79,14 +83,14 @@ describe('TarArchiveResponse', () => { it('should read as archive and files', async () => { const stream = fs.createReadStream('/test-archive.tar.gz'); - const res = new TarArchiveResponse(stream, 'mock-repo/', '/tmp'); + const res = new TarArchiveResponse(stream, 'mock-main/', '/tmp', 'etag'); const buffer = await res.archive(); await expect(res.archive()).rejects.toThrow( 'Response has already been read', ); - const res2 = new TarArchiveResponse(buffer, '', '/tmp'); + const res2 = new TarArchiveResponse(buffer, '', '/tmp', 'etag'); const files = await res2.files(); expect(files).toEqual([ @@ -109,21 +113,26 @@ describe('TarArchiveResponse', () => { it('should extract entire archive into directory', async () => { const stream = fs.createReadStream('/test-archive.tar.gz'); - const res = new TarArchiveResponse(stream, '', '/tmp'); + const res = new TarArchiveResponse(stream, '', '/tmp', 'etag'); const dir = await res.dir(); await expect( - fs.readFile(resolvePath(dir, 'mock-repo/mkdocs.yml'), 'utf8'), + fs.readFile(resolvePath(dir, 'mock-main/mkdocs.yml'), 'utf8'), ).resolves.toBe('site_name: Test\n'); await expect( - fs.readFile(resolvePath(dir, 'mock-repo/docs/index.md'), 'utf8'), + fs.readFile(resolvePath(dir, 'mock-main/docs/index.md'), 'utf8'), ).resolves.toBe('# Test\n'); }); it('should extract archive into directory with a subpath', async () => { const stream = fs.createReadStream('/test-archive.tar.gz'); - const res = new TarArchiveResponse(stream, 'mock-repo/docs/', '/tmp'); + const res = new TarArchiveResponse( + stream, + 'mock-main/docs/', + '/tmp', + 'etag', + ); const dir = await res.dir(); expect(dir).toMatch(/^[\/\\]tmp[\/\\].*$/); @@ -135,8 +144,12 @@ describe('TarArchiveResponse', () => { it('should extract archive into directory with a subpath and filter', async () => { const stream = fs.createReadStream('/test-archive.tar.gz'); - const res = new TarArchiveResponse(stream, 'mock-repo/', '/tmp', path => - path.endsWith('.yml'), + const res = new TarArchiveResponse( + stream, + 'mock-main/', + '/tmp', + 'etag', + path => path.endsWith('.yml'), ); const dir = await res.dir({ targetDir: '/tmp' }); diff --git a/packages/backend-common/src/reading/tree/TarArchiveResponse.ts b/packages/backend-common/src/reading/tree/TarArchiveResponse.ts index 5d18ec7dc6..5927eb75a1 100644 --- a/packages/backend-common/src/reading/tree/TarArchiveResponse.ts +++ b/packages/backend-common/src/reading/tree/TarArchiveResponse.ts @@ -41,6 +41,7 @@ export class TarArchiveResponse implements ReadTreeResponse { private readonly stream: Readable, private readonly subPath: string, private readonly workDir: string, + public readonly etag: string, private readonly filter?: (path: string) => boolean, ) { if (subPath) { @@ -53,6 +54,8 @@ export class TarArchiveResponse implements ReadTreeResponse { ); } } + + this.etag = etag; } // Make sure the input stream is only read once diff --git a/packages/backend-common/src/reading/tree/ZipArchiveResponse.test.ts b/packages/backend-common/src/reading/tree/ZipArchiveResponse.test.ts index 6c2592ffce..b42ec79d81 100644 --- a/packages/backend-common/src/reading/tree/ZipArchiveResponse.test.ts +++ b/packages/backend-common/src/reading/tree/ZipArchiveResponse.test.ts @@ -20,7 +20,7 @@ import { resolve as resolvePath } from 'path'; import { ZipArchiveResponse } from './ZipArchiveResponse'; const archiveData = fs.readFileSync( - resolvePath(__filename, '../../__fixtures__/repo.zip'), + resolvePath(__filename, '../../__fixtures__/mock-main.zip'), ); describe('ZipArchiveResponse', () => { @@ -38,31 +38,35 @@ describe('ZipArchiveResponse', () => { it('should read files', async () => { const stream = fs.createReadStream('/test-archive.zip'); - const res = new ZipArchiveResponse(stream, 'mock-repo/', '/tmp'); + const res = new ZipArchiveResponse(stream, 'mock-main/', '/tmp', 'etag'); const files = await res.files(); expect(files).toEqual([ { - path: 'docs/index.md', + path: 'mkdocs.yml', content: expect.any(Function), }, { - path: 'mkdocs.yml', + path: 'docs/index.md', content: expect.any(Function), }, ]); const contents = await Promise.all(files.map(f => f.content())); expect(contents.map(c => c.toString('utf8').trim())).toEqual([ - '# Test', 'site_name: Test', + '# Test', ]); }); it('should read files with filter', async () => { const stream = fs.createReadStream('/test-archive.zip'); - const res = new ZipArchiveResponse(stream, 'mock-repo/', '/tmp', path => - path.endsWith('.yml'), + const res = new ZipArchiveResponse( + stream, + 'mock-main/', + '/tmp', + 'etag', + path => path.endsWith('.yml'), ); const files = await res.files(); @@ -79,51 +83,56 @@ describe('ZipArchiveResponse', () => { it('should read as archive and files', async () => { const stream = fs.createReadStream('/test-archive.zip'); - const res = new ZipArchiveResponse(stream, 'mock-repo/', '/tmp'); + const res = new ZipArchiveResponse(stream, 'mock-main/', '/tmp', 'etag'); const buffer = await res.archive(); await expect(res.archive()).rejects.toThrow( 'Response has already been read', ); - const res2 = new ZipArchiveResponse(buffer, '', '/tmp'); + const res2 = new ZipArchiveResponse(buffer, '', '/tmp', 'etag'); const files = await res2.files(); expect(files).toEqual([ { - path: 'docs/index.md', + path: 'mkdocs.yml', content: expect.any(Function), }, { - path: 'mkdocs.yml', + path: 'docs/index.md', content: expect.any(Function), }, ]); const contents = await Promise.all(files.map(f => f.content())); expect(contents.map(c => c.toString('utf8').trim())).toEqual([ - '# Test', 'site_name: Test', + '# Test', ]); }); it('should extract entire archive into directory', async () => { const stream = fs.createReadStream('/test-archive.zip'); - const res = new ZipArchiveResponse(stream, '', '/tmp'); + const res = new ZipArchiveResponse(stream, '', '/tmp', 'etag'); const dir = await res.dir(); await expect( - fs.readFile(resolvePath(dir, 'mock-repo/mkdocs.yml'), 'utf8'), + fs.readFile(resolvePath(dir, 'mock-main/mkdocs.yml'), 'utf8'), ).resolves.toBe('site_name: Test\n'); await expect( - fs.readFile(resolvePath(dir, 'mock-repo/docs/index.md'), 'utf8'), + fs.readFile(resolvePath(dir, 'mock-main/docs/index.md'), 'utf8'), ).resolves.toBe('# Test\n'); }); it('should extract archive into directory with a subpath', async () => { const stream = fs.createReadStream('/test-archive.zip'); - const res = new ZipArchiveResponse(stream, 'mock-repo/docs/', '/tmp'); + const res = new ZipArchiveResponse( + stream, + 'mock-main/docs/', + '/tmp', + 'etag', + ); const dir = await res.dir(); expect(dir).toMatch(/^[\/\\]tmp[\/\\].*$/); @@ -135,8 +144,12 @@ describe('ZipArchiveResponse', () => { it('should extract archive into directory with a subpath and filter', async () => { const stream = fs.createReadStream('/test-archive.zip'); - const res = new ZipArchiveResponse(stream, 'mock-repo/', '/tmp', path => - path.endsWith('.yml'), + const res = new ZipArchiveResponse( + stream, + 'mock-main/', + '/tmp', + 'etag', + path => path.endsWith('.yml'), ); const dir = await res.dir({ targetDir: '/tmp' }); diff --git a/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts b/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts index 4106d49a11..07d34faaa3 100644 --- a/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts +++ b/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts @@ -35,6 +35,7 @@ export class ZipArchiveResponse implements ReadTreeResponse { private readonly stream: Readable, private readonly subPath: string, private readonly workDir: string, + public readonly etag: string, private readonly filter?: (path: string) => boolean, ) { if (subPath) { @@ -47,6 +48,8 @@ export class ZipArchiveResponse implements ReadTreeResponse { ); } } + + this.etag = etag; } // Make sure the input stream is only read once diff --git a/packages/backend-common/src/reading/types.ts b/packages/backend-common/src/reading/types.ts index f9dca3e1d5..e98f760d8f 100644 --- a/packages/backend-common/src/reading/types.ts +++ b/packages/backend-common/src/reading/types.ts @@ -32,6 +32,19 @@ export type ReadTreeOptions = { * If no filter is provided all files are extracted. */ filter?(path: string): boolean; + + /** + * An etag can be provided to check whether readTree's response has changed from a previous execution. + * + * In the readTree() response, an etag is returned along with the tree blob. The etag is a unique identifer + * of the tree blob, usually the commit SHA or etag from the target. + * + * When a etag is given in ReadTreeOptions, readTree will first compare the etag against the etag + * on the target branch. If they match, readTree will throw a NotModifiedError indicating that the readTree + * response will not differ from the previous response which included this particular etag. If they mismatch, + * readTree will return the rest of ReadTreeResponse along with a new etag. + */ + etag?: string; }; /** @@ -70,5 +83,14 @@ export type ReadTreeResponseDirOptions = { export type ReadTreeResponse = { files(): Promise; archive(): Promise; + + /** + * dir() extracts the tree response into a directory and returns the path of the directory. + */ dir(options?: ReadTreeResponseDirOptions): Promise; + + /** + * A unique identifer of the tree blob, usually the commit SHA or etag from the target. + */ + etag: string; }; diff --git a/packages/backend/CHANGELOG.md b/packages/backend/CHANGELOG.md index e9246ab533..1d840d8761 100644 --- a/packages/backend/CHANGELOG.md +++ b/packages/backend/CHANGELOG.md @@ -1,5 +1,39 @@ # example-backend +## 0.2.12 + +### Patch Changes + +- Updated dependencies [def2307f3] +- Updated dependencies [d54857099] +- Updated dependencies [0b135e7e0] +- Updated dependencies [318a6af9f] +- Updated dependencies [294a70cab] +- Updated dependencies [ac7be581a] +- Updated dependencies [0ea032763] +- Updated dependencies [5345a1f98] +- Updated dependencies [ed6baab66] +- Updated dependencies [ad838c02f] +- Updated dependencies [a5e27d5c1] +- Updated dependencies [0643a3336] +- Updated dependencies [a2291d7cc] +- Updated dependencies [f9ba00a1c] +- Updated dependencies [09a370426] +- Updated dependencies [a93f42213] + - @backstage/catalog-model@0.7.0 + - @backstage/plugin-catalog-backend@0.5.4 + - @backstage/plugin-kubernetes-backend@0.2.5 + - @backstage/backend-common@0.5.0 + - @backstage/plugin-scaffolder-backend@0.5.0 + - @backstage/plugin-techdocs-backend@0.5.4 + - @backstage/plugin-auth-backend@0.2.11 + - example-app@0.2.12 + - @backstage/plugin-kafka-backend@0.1.1 + - @backstage/plugin-app-backend@0.3.4 + - @backstage/plugin-graphql-backend@0.1.5 + - @backstage/plugin-proxy-backend@0.2.4 + - @backstage/plugin-rollbar-backend@0.1.7 + ## 0.2.11 ### Patch Changes diff --git a/packages/backend/package.json b/packages/backend/package.json index 243ebf2ce3..269089a400 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -1,6 +1,6 @@ { "name": "example-backend", - "version": "0.2.11", + "version": "0.2.12", "main": "dist/index.cjs.js", "types": "src/index.ts", "license": "Apache-2.0", @@ -27,23 +27,24 @@ "migrate:create": "knex migrate:make -x ts" }, "dependencies": { - "@backstage/backend-common": "^0.4.3", - "@backstage/catalog-model": "^0.6.1", + "@backstage/backend-common": "^0.5.0", + "@backstage/catalog-model": "^0.7.0", "@backstage/config": "^0.1.2", - "@backstage/plugin-app-backend": "^0.3.3", - "@backstage/plugin-auth-backend": "^0.2.10", - "@backstage/plugin-catalog-backend": "^0.5.3", - "@backstage/plugin-graphql-backend": "^0.1.4", - "@backstage/plugin-kubernetes-backend": "^0.2.4", - "@backstage/plugin-proxy-backend": "^0.2.3", - "@backstage/plugin-rollbar-backend": "^0.1.5", - "@backstage/plugin-scaffolder-backend": "^0.4.1", - "@backstage/plugin-techdocs-backend": "^0.5.3", + "@backstage/plugin-app-backend": "^0.3.4", + "@backstage/plugin-auth-backend": "^0.2.11", + "@backstage/plugin-catalog-backend": "^0.5.4", + "@backstage/plugin-graphql-backend": "^0.1.5", + "@backstage/plugin-kubernetes-backend": "^0.2.5", + "@backstage/plugin-kafka-backend": "^0.1.1", + "@backstage/plugin-proxy-backend": "^0.2.4", + "@backstage/plugin-rollbar-backend": "^0.1.7", + "@backstage/plugin-scaffolder-backend": "^0.5.0", + "@backstage/plugin-techdocs-backend": "^0.5.4", "@gitbeaker/node": "^28.0.2", "@octokit/rest": "^18.0.12", "azure-devops-node-api": "^10.1.1", "dockerode": "^3.2.1", - "example-app": "^0.2.8", + "example-app": "^0.2.12", "express": "^4.17.1", "express-promise-router": "^3.0.3", "knex": "^0.21.6", @@ -53,7 +54,7 @@ "winston": "^3.2.1" }, "devDependencies": { - "@backstage/cli": "^0.4.6", + "@backstage/cli": "^0.4.7", "@types/dockerode": "^3.2.1", "@types/express": "^4.17.6", "@types/express-serve-static-core": "^4.17.5" diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index 68cc170901..81d0bb96d2 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -38,6 +38,7 @@ import healthcheck from './plugins/healthcheck'; import auth from './plugins/auth'; import catalog from './plugins/catalog'; import kubernetes from './plugins/kubernetes'; +import kafka from './plugins/kafka'; import rollbar from './plugins/rollbar'; import scaffolder from './plugins/scaffolder'; import proxy from './plugins/proxy'; @@ -77,6 +78,7 @@ async function main() { const rollbarEnv = useHotMemoize(module, () => createEnv('rollbar')); const techdocsEnv = useHotMemoize(module, () => createEnv('techdocs')); const kubernetesEnv = useHotMemoize(module, () => createEnv('kubernetes')); + const kafkaEnv = useHotMemoize(module, () => createEnv('kafka')); const graphqlEnv = useHotMemoize(module, () => createEnv('graphql')); const appEnv = useHotMemoize(module, () => createEnv('app')); @@ -87,6 +89,7 @@ async function main() { apiRouter.use('/auth', await auth(authEnv)); apiRouter.use('/techdocs', await techdocs(techdocsEnv)); apiRouter.use('/kubernetes', await kubernetes(kubernetesEnv)); + apiRouter.use('/kafka', await kafka(kafkaEnv)); apiRouter.use('/proxy', await proxy(proxyEnv)); apiRouter.use('/graphql', await graphql(graphqlEnv)); apiRouter.use(notFoundHandler()); diff --git a/packages/backend/src/plugins/kafka.ts b/packages/backend/src/plugins/kafka.ts new file mode 100644 index 0000000000..e65ce6719c --- /dev/null +++ b/packages/backend/src/plugins/kafka.ts @@ -0,0 +1,25 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { createRouter } from '@backstage/plugin-kafka-backend'; +import { PluginEnvironment } from '../types'; + +export default async function createPlugin({ + logger, + config, +}: PluginEnvironment) { + return await createRouter({ logger, config }); +} diff --git a/packages/backend/src/plugins/scaffolder.ts b/packages/backend/src/plugins/scaffolder.ts index 5d36d508a5..4e2257a46c 100644 --- a/packages/backend/src/plugins/scaffolder.ts +++ b/packages/backend/src/plugins/scaffolder.ts @@ -34,6 +34,7 @@ export default async function createPlugin({ const cookiecutterTemplater = new CookieCutter(); const craTemplater = new CreateReactAppTemplater(); const templaters = new Templaters(); + templaters.register('cookiecutter', cookiecutterTemplater); templaters.register('cra', craTemplater); diff --git a/packages/catalog-client/CHANGELOG.md b/packages/catalog-client/CHANGELOG.md index c99f72e376..c8042b72e9 100644 --- a/packages/catalog-client/CHANGELOG.md +++ b/packages/catalog-client/CHANGELOG.md @@ -1,5 +1,13 @@ # @backstage/catalog-client +## 0.3.5 + +### Patch Changes + +- Updated dependencies [def2307f3] +- Updated dependencies [a93f42213] + - @backstage/catalog-model@0.7.0 + ## 0.3.4 ### Patch Changes diff --git a/packages/catalog-client/package.json b/packages/catalog-client/package.json index c878119a35..0b68446463 100644 --- a/packages/catalog-client/package.json +++ b/packages/catalog-client/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/catalog-client", - "version": "0.3.4", + "version": "0.3.5", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -29,12 +29,12 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@backstage/catalog-model": "^0.6.0", + "@backstage/catalog-model": "^0.7.0", "@backstage/config": "^0.1.2", "cross-fetch": "^3.0.6" }, "devDependencies": { - "@backstage/cli": "^0.4.2", + "@backstage/cli": "^0.4.7", "@types/jest": "^26.0.7", "msw": "^0.21.2" }, diff --git a/packages/catalog-model/CHANGELOG.md b/packages/catalog-model/CHANGELOG.md index 97f54bbdd2..6a081c3336 100644 --- a/packages/catalog-model/CHANGELOG.md +++ b/packages/catalog-model/CHANGELOG.md @@ -1,5 +1,38 @@ # @backstage/catalog-model +## 0.7.0 + +### Minor Changes + +- a93f42213: The catalog no longer attempts to merge old and new annotations, when updating an entity from a remote location. This was a behavior that was copied from kubernetes, and catered to use cases where you wanted to use HTTP POST to update an entity in-place, outside of what the refresh loop does. This has proved to be a mistake, because as a side effect, the refresh loop effectively is unable to ever delete annotations when they are removed from source YAML. This is obviously a breaking change, but we believe that this is not a behavior that is relied upon in the wild, and it has never been an actually supported use flow of the catalog. We therefore choose to break the behavior outright, and instead just store updated annotations verbatim - just like we already do for example for labels + +### Patch Changes + +- def2307f3: Adds a `backstage.io/managed-by-origin-location` annotation to all entities. It links to the + location that was registered to the catalog and which emitted this entity. It has a different + semantic than the existing `backstage.io/managed-by-location` annotation, which tells the direct + parent location that created this entity. + + Consider this example: The Backstage operator adds a location of type `github-org` in the + `app-config.yaml`. This setting will be added to a `bootstrap:boostrap` location. The processor + discovers the entities in the following branch + `Location bootstrap:bootstrap -> Location github-org:… -> User xyz`. The user `xyz` will be: + + ```yaml + apiVersion: backstage.io/v1alpha1 + kind: User + metadata: + name: xyz + annotations: + # This entity was added by the 'github-org:…' location + backstage.io/managed-by-location: github-org:… + # The entity was added because the 'bootstrap:boostrap' was added to the catalog + backstage.io/managed-by-origin-location: bootstrap:bootstrap + # ... + spec: + # ... + ``` + ## 0.6.1 ### Patch Changes diff --git a/packages/catalog-model/examples/all-components.yaml b/packages/catalog-model/examples/all-components.yaml index 5db5825b20..9471b810a1 100644 --- a/packages/catalog-model/examples/all-components.yaml +++ b/packages/catalog-model/examples/all-components.yaml @@ -15,4 +15,6 @@ spec: - ./components/www-artist-component.yaml - ./components/shuffle-api-component.yaml - ./components/wayback-archive-component.yaml + - ./components/wayback-archive-ingestion-component.yaml + - ./components/wayback-archive-storage-component.yaml - ./components/wayback-search-component.yaml diff --git a/packages/catalog-model/examples/components/wayback-archive-ingestion-component.yaml b/packages/catalog-model/examples/components/wayback-archive-ingestion-component.yaml new file mode 100644 index 0000000000..4f870ac831 --- /dev/null +++ b/packages/catalog-model/examples/components/wayback-archive-ingestion-component.yaml @@ -0,0 +1,10 @@ +apiVersion: backstage.io/v1alpha1 +kind: Component +metadata: + name: wayback-archive-ingestion + description: Ingestion subsystem of the Wayback Archive +spec: + type: service + lifecycle: production + owner: team-d + subcomponentOf: wayback-archive diff --git a/packages/catalog-model/examples/components/wayback-archive-storage-component.yaml b/packages/catalog-model/examples/components/wayback-archive-storage-component.yaml new file mode 100644 index 0000000000..78d67258d7 --- /dev/null +++ b/packages/catalog-model/examples/components/wayback-archive-storage-component.yaml @@ -0,0 +1,10 @@ +apiVersion: backstage.io/v1alpha1 +kind: Component +metadata: + name: wayback-archive-storage + description: Storage subsystem of the Wayback Archive +spec: + type: service + lifecycle: production + owner: team-a + subcomponentOf: wayback-archive diff --git a/packages/catalog-model/package.json b/packages/catalog-model/package.json index 01ae2e45b7..e03ce8c465 100644 --- a/packages/catalog-model/package.json +++ b/packages/catalog-model/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/catalog-model", - "version": "0.6.1", + "version": "0.7.0", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -38,7 +38,7 @@ "yup": "^0.29.3" }, "devDependencies": { - "@backstage/cli": "^0.4.6", + "@backstage/cli": "^0.4.7", "@types/express": "^4.17.6", "@types/jest": "^26.0.7", "@types/lodash": "^4.14.151", diff --git a/packages/catalog-model/src/location/annotation.ts b/packages/catalog-model/src/location/annotation.ts index 371d095685..93f2fabea4 100644 --- a/packages/catalog-model/src/location/annotation.ts +++ b/packages/catalog-model/src/location/annotation.ts @@ -15,3 +15,5 @@ */ export const LOCATION_ANNOTATION = 'backstage.io/managed-by-location'; +export const ORIGIN_LOCATION_ANNOTATION = + 'backstage.io/managed-by-origin-location'; diff --git a/packages/catalog-model/src/location/index.ts b/packages/catalog-model/src/location/index.ts index ce64b988a6..8fd516120a 100644 --- a/packages/catalog-model/src/location/index.ts +++ b/packages/catalog-model/src/location/index.ts @@ -20,4 +20,4 @@ export { locationSpecSchema, analyzeLocationSchema, } from './validation'; -export { LOCATION_ANNOTATION } from './annotation'; +export { LOCATION_ANNOTATION, ORIGIN_LOCATION_ANNOTATION } from './annotation'; diff --git a/packages/cli/CHANGELOG.md b/packages/cli/CHANGELOG.md index 738b56d82d..7998d408e0 100644 --- a/packages/cli/CHANGELOG.md +++ b/packages/cli/CHANGELOG.md @@ -1,5 +1,11 @@ # @backstage/cli +## 0.4.7 + +### Patch Changes + +- b604a9d41: Append `-credentials.yaml` to credentials file generated by `backstage-cli create-github-app` and display warning about sensitive contents. + ## 0.4.6 ### Patch Changes diff --git a/packages/cli/package.json b/packages/cli/package.json index 4605a6bdff..e7d4379b5d 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,7 +1,7 @@ { "name": "@backstage/cli", "description": "CLI for developing Backstage plugins and apps", - "version": "0.4.6", + "version": "0.4.7", "private": false, "publishConfig": { "access": "public" @@ -113,10 +113,10 @@ "yn": "^4.0.0" }, "devDependencies": { - "@backstage/backend-common": "^0.4.3", + "@backstage/backend-common": "^0.5.0", "@backstage/config": "^0.1.2", - "@backstage/core": "^0.4.4", - "@backstage/dev-utils": "^0.1.7", + "@backstage/core": "^0.5.0", + "@backstage/dev-utils": "^0.1.8", "@backstage/test-utils": "^0.1.6", "@backstage/theme": "^0.2.2", "@types/diff": "^4.0.2", diff --git a/packages/core-api/src/plugin/Plugin.tsx b/packages/core-api/src/plugin/Plugin.tsx index dd6d7960ac..cc168707be 100644 --- a/packages/core-api/src/plugin/Plugin.tsx +++ b/packages/core-api/src/plugin/Plugin.tsx @@ -68,9 +68,6 @@ export class PluginImpl< options, }); }, - registerRoute(path, component, options) { - outputs.push({ type: 'legacy-route', path, component, options }); - }, }, featureFlags: { register(name) { diff --git a/packages/core-api/src/plugin/types.ts b/packages/core-api/src/plugin/types.ts index 94bd4e8172..711246f1f4 100644 --- a/packages/core-api/src/plugin/types.ts +++ b/packages/core-api/src/plugin/types.ts @@ -99,26 +99,22 @@ export type PluginConfig< }; export type PluginHooks = { + /** + * @deprecated All router hooks have been deprecated + */ router: RouterHooks; featureFlags: FeatureFlagsHooks; }; export type RouterHooks = { + /** + * @deprecated Use a routable extension instead, see https://backstage.io/docs/plugins/composability#porting-existing-plugins + */ addRoute( target: RouteRef, Component: ComponentType, options?: RouteOptions, ): void; - - /** - * @deprecated See the `addRoute` method - * @see https://github.com/backstage/backstage/issues/418 - */ - registerRoute( - path: RoutePath, - Component: ComponentType, - options?: RouteOptions, - ): void; }; export type FeatureFlagsHooks = { diff --git a/packages/core/CHANGELOG.md b/packages/core/CHANGELOG.md index b9ea855c83..0fea08e539 100644 --- a/packages/core/CHANGELOG.md +++ b/packages/core/CHANGELOG.md @@ -1,5 +1,65 @@ # @backstage/core +## 0.5.0 + +### Minor Changes + +- efd6ef753: Removed `InfoCard` variant `height100`, originally deprecated in [#2826](https://github.com/backstage/backstage/pull/2826). + + If your component still relies on this variant, simply replace it with `gridItem`. + +- a187b8ad0: Removed deprecated `router.registerRoute` method in `createPlugin`. + + Deprecated `router.addRoute` method in `createPlugin`. + + Replace usage of the above two components with a routable extension. + + For example, given the following: + + ```ts + import { createPlugin } from '@backstage/core'; + import { MyPage } from './components/MyPage'; + import { rootRoute } from './routes'; + + export const plugin = createPlugin({ + id: 'my-plugin', + register({ router }) { + router.addRoute(rootRoute, MyPage); + }, + }); + ``` + + Migrate to + + ```ts + import { createPlugin, createRoutableExtension } from '@backstage/core'; + import { rootRoute } from './routes'; + + export const plugin = createPlugin({ + id: 'my-plugin', + routes: { + root: rootRoute, + }, + }); + + export const MyPage = plugin.provide( + createRoutableExtension({ + component: () => import('./components/MyPage').then(m => m.MyPage), + mountPoint: rootRoute, + }), + ); + ``` + + And then use `MyPage` like this in the app: + + ```tsx + + ... + }> + ... + + ``` + ## 0.4.4 ### Patch Changes diff --git a/packages/core/package.json b/packages/core/package.json index d86ea3b240..a937502247 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -1,7 +1,7 @@ { "name": "@backstage/core", "description": "Core API used by Backstage plugins and apps", - "version": "0.4.4", + "version": "0.5.0", "private": false, "publishConfig": { "access": "public", @@ -46,7 +46,7 @@ "d3-zoom": "^2.0.0", "dagre": "^0.8.5", "qs": "^6.9.4", - "immer": "^7.0.9", + "immer": "^8.0.1", "lodash": "^4.17.15", "material-table": "^1.69.1", "prop-types": "^15.7.2", @@ -65,7 +65,7 @@ "zen-observable": "^0.8.15" }, "devDependencies": { - "@backstage/cli": "^0.4.6", + "@backstage/cli": "^0.4.7", "@backstage/test-utils": "^0.1.6", "@testing-library/jest-dom": "^5.10.1", "@testing-library/react": "^10.4.1", diff --git a/packages/create-app/CHANGELOG.md b/packages/create-app/CHANGELOG.md index 8dba8d8fc1..e33d8cc633 100644 --- a/packages/create-app/CHANGELOG.md +++ b/packages/create-app/CHANGELOG.md @@ -1,5 +1,250 @@ # @backstage/create-app +## 0.3.6 + +### Patch Changes + +- d3947caf3: Fix accidental dependency on non-existent dependencies. +- Updated dependencies [a4e636c8f] +- Updated dependencies [099c5cf4f] +- Updated dependencies [0ea002378] +- Updated dependencies [a08db734c] + - @backstage/plugin-catalog@0.2.13 + - @backstage/plugin-scaffolder-backend@0.5.1 + +## 1.0.0 + +### Minor Changes + +- ed6baab66: - Deprecating the `scaffolder.${provider}.token` auth duplication and favoring `integrations.${provider}` instead. If you receive deprecation warnings your config should change like the following: + + ```yaml + scaffolder: + github: + token: + $env: GITHUB_TOKEN + visibility: public + ``` + + To something that looks like this: + + ```yaml + integration: + github: + - host: github.com + token: + $env: GITHUB_TOKEN + scaffolder: + github: + visibility: public + ``` + + You can also configure multiple different hosts under the `integration` config like the following: + + ```yaml + integration: + github: + - host: github.com + token: + $env: GITHUB_TOKEN + - host: ghe.mycompany.com + token: + $env: GITHUB_ENTERPRISE_TOKEN + ``` + + This of course is the case for all the providers respectively. + + - Adding support for cross provider scaffolding, you can now create repositories in for example Bitbucket using a template residing in GitHub. + + - Fix GitLab scaffolding so that it returns a `catalogInfoUrl` which automatically imports the project into the catalog. + + - The `Store Path` field on the `scaffolder` frontend has now changed so that you require the full URL to the desired destination repository. + + `backstage/new-repository` would become `https://github.com/backstage/new-repository` if provider was GitHub for example. + +### Patch Changes + +- a284f5bc1: Due to a package name change from `@kyma-project/asyncapi-react` to + `@asyncapi/react-component` the jest configuration in the root `package.json` + has to be updated: + + ```diff + "jest": { + "transformModules": [ + - "@kyma-project/asyncapi-react + + "@asyncapi/react-component" + ] + } + ``` + +- 89278acab: Migrate to using `FlatRoutes` from `@backstage/core` for the root app routes. + + This is the first step in migrating applications as mentioned here: https://backstage.io/docs/plugins/composability#porting-existing-apps. + + To apply this change to an existing app, switch out the `Routes` component from `react-router` to `FlatRoutes` from `@backstage/core`. + This also allows you to remove any `/*` suffixes on the route paths. For example: + + ```diff + import { + OAuthRequestDialog, + SidebarPage, + createRouteRef, + + FlatRoutes, + } from '@backstage/core'; + import { AppSidebar } from './sidebar'; + -import { Route, Routes, Navigate } from 'react-router'; + +import { Route, Navigate } from 'react-router'; + import { Router as CatalogRouter } from '@backstage/plugin-catalog'; + ... + + - + + + ... + } + /> + - } /> + + } /> + ... + } /> + - + + + + ``` + +- 26d3b24f3: fix routing and config for user-settings plugin + + To make the corresponding change in your local app, add the following in your App.tsx + + ``` + import { Router as SettingsRouter } from '@backstage/plugin-user-settings'; + ... + } /> + ``` + + and the following to your plugins.ts: + + ``` + export { plugin as UserSettings } from '@backstage/plugin-user-settings'; + ``` + +- 92dbbcedd: Add `*-credentials.yaml` to gitignore to prevent accidental commits of sensitive credential information. + + To apply this change to an existing installation, add these lines to your `.gitignore` + + ```gitignore + # Sensitive credentials + *-credentials.yaml + ``` + +- d176671d1: use `fromConfig` for all scaffolder helpers, and use the url protocol for app-config location entries. + + To apply this change to your local installation, replace the contents of your `packages/backend/src/plugins/scaffolder.ts` with the following contents: + + ```ts + import { + CookieCutter, + createRouter, + Preparers, + Publishers, + CreateReactAppTemplater, + Templaters, + CatalogEntityClient, + } from '@backstage/plugin-scaffolder-backend'; + import { SingleHostDiscovery } from '@backstage/backend-common'; + import type { PluginEnvironment } from '../types'; + import Docker from 'dockerode'; + + export default async function createPlugin({ + logger, + config, + }: PluginEnvironment) { + const cookiecutterTemplater = new CookieCutter(); + const craTemplater = new CreateReactAppTemplater(); + const templaters = new Templaters(); + templaters.register('cookiecutter', cookiecutterTemplater); + templaters.register('cra', craTemplater); + + const preparers = await Preparers.fromConfig(config, { logger }); + const publishers = await Publishers.fromConfig(config, { logger }); + + const dockerClient = new Docker(); + + const discovery = SingleHostDiscovery.fromConfig(config); + const entityClient = new CatalogEntityClient({ discovery }); + + return await createRouter({ + preparers, + templaters, + publishers, + logger, + config, + dockerClient, + entityClient, + }); + } + ``` + + This will ensure that the `scaffolder-backend` package can add handlers for the `url` protocol which is becoming the standard when registering entities in the `catalog` + +- 9d1d1138e: Ensured that versions bumps of packages used in the app template trigger a release of this package when needed. +- db05f7a35: Remove the `@types/helmet` dev dependency from the app template. This + dependency is now unused as the package `helmet` brings its own types. + + To update your existing app, simply remove the `@types/helmet` dependency from + the `package.json` of your backend package. + +- Updated dependencies [def2307f3] +- Updated dependencies [46bba09ea] +- Updated dependencies [efd6ef753] +- Updated dependencies [0b135e7e0] +- Updated dependencies [593632f07] +- Updated dependencies [2b514d532] +- Updated dependencies [318a6af9f] +- Updated dependencies [33846acfc] +- Updated dependencies [294a70cab] +- Updated dependencies [b604a9d41] +- Updated dependencies [ac7be581a] +- Updated dependencies [a187b8ad0] +- Updated dependencies [0ea032763] +- Updated dependencies [8855f61f6] +- Updated dependencies [5345a1f98] +- Updated dependencies [ed6baab66] +- Updated dependencies [ad838c02f] +- Updated dependencies [f04db53d7] +- Updated dependencies [a5e27d5c1] +- Updated dependencies [0643a3336] +- Updated dependencies [debf359b5] +- Updated dependencies [a2291d7cc] +- Updated dependencies [f9ba00a1c] +- Updated dependencies [09a370426] +- Updated dependencies [a93f42213] + - @backstage/catalog-model@0.7.0 + - @backstage/plugin-catalog-backend@0.5.4 + - @backstage/plugin-github-actions@0.3.0 + - @backstage/core@0.5.0 + - @backstage/backend-common@0.5.0 + - @backstage/plugin-catalog@0.2.12 + - @backstage/plugin-catalog-import@0.3.5 + - @backstage/cli@0.4.7 + - @backstage/plugin-api-docs@0.4.3 + - @backstage/plugin-scaffolder@0.4.0 + - @backstage/plugin-scaffolder-backend@0.5.0 + - @backstage/plugin-techdocs@0.5.4 + - @backstage/plugin-techdocs-backend@0.5.4 + - @backstage/plugin-auth-backend@0.2.11 + - @backstage/plugin-lighthouse@0.2.8 + - @backstage/plugin-circleci@0.2.6 + - @backstage/plugin-search@0.2.6 + - @backstage/plugin-explore@0.2.3 + - @backstage/plugin-tech-radar@0.3.3 + - @backstage/plugin-user-settings@0.2.4 + - @backstage/plugin-app-backend@0.3.4 + - @backstage/plugin-proxy-backend@0.2.4 + - @backstage/plugin-rollbar-backend@0.1.7 + ## 0.3.5 ### Patch Changes diff --git a/packages/create-app/package.json b/packages/create-app/package.json index 0c703bcdac..60dd150566 100644 --- a/packages/create-app/package.json +++ b/packages/create-app/package.json @@ -1,7 +1,7 @@ { "name": "@backstage/create-app", "description": "Create app package for Backstage", - "version": "0.3.5", + "version": "0.3.6", "private": false, "publishConfig": { "access": "public" @@ -37,38 +37,40 @@ "recursive-readdir": "^2.2.2" }, "devDependencies": { - "@backstage/backend-common": "^0.4.3", - "@backstage/catalog-model": "^0.6.1", - "@backstage/cli": "^0.4.6", - "@backstage/config": "^0.1.2", - "@backstage/core": "^0.4.4", - "@backstage/plugin-api-docs": "^0.4.2", - "@backstage/plugin-app-backend": "^0.3.3", - "@backstage/plugin-auth-backend": "^0.2.10", - "@backstage/plugin-catalog": "^0.2.11", - "@backstage/plugin-catalog-backend": "^0.5.3", - "@backstage/plugin-catalog-import": "^0.3.4", - "@backstage/plugin-circleci": "^0.2.5", - "@backstage/plugin-explore": "^0.2.2", - "@backstage/plugin-github-actions": "^0.2.7", - "@backstage/plugin-lighthouse": "^0.2.7", - "@backstage/plugin-proxy-backend": "^0.2.3", - "@backstage/plugin-rollbar-backend": "^0.1.6", - "@backstage/plugin-scaffolder": "^0.3.6", - "@backstage/plugin-search": "^0.2.5", - "@backstage/plugin-scaffolder-backend": "^0.4.1", - "@backstage/plugin-tech-radar": "^0.3.2", - "@backstage/plugin-techdocs": "^0.5.3", - "@backstage/plugin-techdocs-backend": "^0.5.3", - "@backstage/plugin-user-settings": "^0.2.3", - "@backstage/test-utils": "^0.1.6", - "@backstage/theme": "^0.2.2", "@types/fs-extra": "^9.0.1", "@types/inquirer": "^7.3.1", "@types/react-dev-utils": "^9.0.4", "@types/recursive-readdir": "^2.2.0", "ts-node": "^8.6.2" }, + "peerDependencies": { + "@backstage/backend-common": "^0.5.0", + "@backstage/catalog-model": "^0.7.0", + "@backstage/cli": "^0.4.7", + "@backstage/config": "^0.1.2", + "@backstage/core": "^0.5.0", + "@backstage/plugin-api-docs": "^0.4.3", + "@backstage/plugin-app-backend": "^0.3.4", + "@backstage/plugin-auth-backend": "^0.2.11", + "@backstage/plugin-catalog": "^0.2.13", + "@backstage/plugin-catalog-backend": "^0.5.4", + "@backstage/plugin-catalog-import": "^0.3.5", + "@backstage/plugin-circleci": "^0.2.6", + "@backstage/plugin-explore": "^0.2.3", + "@backstage/plugin-github-actions": "^0.3.0", + "@backstage/plugin-lighthouse": "^0.2.8", + "@backstage/plugin-proxy-backend": "^0.2.4", + "@backstage/plugin-rollbar-backend": "^0.1.7", + "@backstage/plugin-scaffolder": "^0.4.0", + "@backstage/plugin-search": "^0.2.6", + "@backstage/plugin-scaffolder-backend": "^0.5.1", + "@backstage/plugin-tech-radar": "^0.3.3", + "@backstage/plugin-techdocs": "^0.5.4", + "@backstage/plugin-techdocs-backend": "^0.5.4", + "@backstage/plugin-user-settings": "^0.2.4", + "@backstage/test-utils": "^0.1.6", + "@backstage/theme": "^0.2.2" + }, "nodemonConfig": { "watch": "./src", "exec": "bin/backstage-create-app", diff --git a/packages/create-app/src/lib/versions.ts b/packages/create-app/src/lib/versions.ts index 5cbf6c77ee..a703fb9305 100644 --- a/packages/create-app/src/lib/versions.ts +++ b/packages/create-app/src/lib/versions.ts @@ -14,47 +14,49 @@ * limitations under the License. */ -/* eslint-disable import/no-extraneous-dependencies,monorepo/no-internal-import */ +/* eslint-disable monorepo/no-relative-import */ /* This is a list of all packages used by the template. If dependencies are added or removed, this list should be updated as well. -The list, and the accompanying devDependencies entries, are here to ensure correct versioning +The list, and the accompanying peerDependencies entries, are here to ensure correct versioning and bumping of this package. Without this list the version would not be bumped unless we manually trigger a release. This does not create an actual dependency on these packages and does not bring in any code. +Relative imports are used rather than package imports to make sure the packages aren't externalized. Rollup will extract the value of the version field in each package at build time without leaving any imports in place. */ -import { version as backendCommon } from '@backstage/backend-common/package.json'; -import { version as catalogModel } from '@backstage/catalog-model/package.json'; -import { version as cli } from '@backstage/cli/package.json'; -import { version as config } from '@backstage/config/package.json'; -import { version as core } from '@backstage/core/package.json'; -import { version as pluginApiDocs } from '@backstage/plugin-api-docs/package.json'; -import { version as pluginAppBackend } from '@backstage/plugin-app-backend/package.json'; -import { version as pluginAuthBackend } from '@backstage/plugin-auth-backend/package.json'; -import { version as pluginCatalog } from '@backstage/plugin-catalog/package.json'; -import { version as pluginCatalogBackend } from '@backstage/plugin-catalog-backend/package.json'; -import { version as pluginCatalogImport } from '@backstage/plugin-catalog-import/package.json'; -import { version as pluginCircleci } from '@backstage/plugin-circleci/package.json'; -import { version as pluginExplore } from '@backstage/plugin-explore/package.json'; -import { version as pluginGithubActions } from '@backstage/plugin-github-actions/package.json'; -import { version as pluginLighthouse } from '@backstage/plugin-lighthouse/package.json'; -import { version as pluginProxyBackend } from '@backstage/plugin-proxy-backend/package.json'; -import { version as pluginRollbarBackend } from '@backstage/plugin-rollbar-backend/package.json'; -import { version as pluginScaffolder } from '@backstage/plugin-scaffolder/package.json'; -import { version as pluginScaffolderBackend } from '@backstage/plugin-scaffolder-backend/package.json'; -import { version as pluginSearch } from '@backstage/plugin-search/package.json'; -import { version as pluginTechRadar } from '@backstage/plugin-tech-radar/package.json'; -import { version as pluginTechdocs } from '@backstage/plugin-techdocs/package.json'; -import { version as pluginTechdocsBackend } from '@backstage/plugin-techdocs-backend/package.json'; -import { version as pluginUserSettings } from '@backstage/plugin-user-settings/package.json'; -import { version as testUtils } from '@backstage/test-utils/package.json'; -import { version as theme } from '@backstage/theme/package.json'; +import { version as backendCommon } from '../../../backend-common/package.json'; +import { version as catalogModel } from '../../../catalog-model/package.json'; +import { version as cli } from '../../../cli/package.json'; +import { version as config } from '../../../config/package.json'; +import { version as core } from '../../../core/package.json'; +import { version as testUtils } from '../../../test-utils/package.json'; +import { version as theme } from '../../../theme/package.json'; + +import { version as pluginApiDocs } from '../../../../plugins/api-docs/package.json'; +import { version as pluginAppBackend } from '../../../../plugins/app-backend/package.json'; +import { version as pluginAuthBackend } from '../../../../plugins/auth-backend/package.json'; +import { version as pluginCatalog } from '../../../../plugins/catalog/package.json'; +import { version as pluginCatalogBackend } from '../../../../plugins/catalog-backend/package.json'; +import { version as pluginCatalogImport } from '../../../../plugins/catalog-import/package.json'; +import { version as pluginCircleci } from '../../../../plugins/circleci/package.json'; +import { version as pluginExplore } from '../../../../plugins/explore/package.json'; +import { version as pluginGithubActions } from '../../../../plugins/github-actions/package.json'; +import { version as pluginLighthouse } from '../../../../plugins/lighthouse/package.json'; +import { version as pluginProxyBackend } from '../../../../plugins/proxy-backend/package.json'; +import { version as pluginRollbarBackend } from '../../../../plugins/rollbar-backend/package.json'; +import { version as pluginScaffolder } from '../../../../plugins/scaffolder/package.json'; +import { version as pluginScaffolderBackend } from '../../../../plugins/scaffolder-backend/package.json'; +import { version as pluginSearch } from '../../../../plugins/search/package.json'; +import { version as pluginTechRadar } from '../../../../plugins/tech-radar/package.json'; +import { version as pluginTechdocs } from '../../../../plugins/techdocs/package.json'; +import { version as pluginTechdocsBackend } from '../../../../plugins/techdocs-backend/package.json'; +import { version as pluginUserSettings } from '../../../../plugins/user-settings/package.json'; export const packageVersions = { '@backstage/backend-common': backendCommon, diff --git a/packages/create-app/templates/default-app/package.json.hbs b/packages/create-app/templates/default-app/package.json.hbs index 1353e33072..3fa350fa7c 100644 --- a/packages/create-app/templates/default-app/package.json.hbs +++ b/packages/create-app/templates/default-app/package.json.hbs @@ -45,7 +45,7 @@ }, "jest": { "transformModules": [ - "@kyma-project/asyncapi-react" + "@asyncapi/react-component" ] } } diff --git a/packages/create-app/templates/default-app/packages/backend/src/plugins/scaffolder.ts b/packages/create-app/templates/default-app/packages/backend/src/plugins/scaffolder.ts index 196d48ec78..c8bd3e5012 100644 --- a/packages/create-app/templates/default-app/packages/backend/src/plugins/scaffolder.ts +++ b/packages/create-app/templates/default-app/packages/backend/src/plugins/scaffolder.ts @@ -18,6 +18,7 @@ export default async function createPlugin({ const cookiecutterTemplater = new CookieCutter(); const craTemplater = new CreateReactAppTemplater(); const templaters = new Templaters(); + templaters.register('cookiecutter', cookiecutterTemplater); templaters.register('cra', craTemplater); diff --git a/packages/dev-utils/CHANGELOG.md b/packages/dev-utils/CHANGELOG.md index 1938754169..c6c740bf39 100644 --- a/packages/dev-utils/CHANGELOG.md +++ b/packages/dev-utils/CHANGELOG.md @@ -1,5 +1,20 @@ # @backstage/dev-utils +## 0.1.8 + +### Patch Changes + +- Updated dependencies [def2307f3] +- Updated dependencies [efd6ef753] +- Updated dependencies [593632f07] +- Updated dependencies [33846acfc] +- Updated dependencies [a187b8ad0] +- Updated dependencies [f04db53d7] +- Updated dependencies [a93f42213] + - @backstage/catalog-model@0.7.0 + - @backstage/core@0.5.0 + - @backstage/plugin-catalog@0.2.12 + ## 0.1.7 ### Patch Changes diff --git a/packages/dev-utils/package.json b/packages/dev-utils/package.json index 785607c6d3..629bc7c87e 100644 --- a/packages/dev-utils/package.json +++ b/packages/dev-utils/package.json @@ -1,7 +1,7 @@ { "name": "@backstage/dev-utils", "description": "Utilities for developing Backstage plugins.", - "version": "0.1.7", + "version": "0.1.8", "private": false, "publishConfig": { "access": "public", @@ -29,9 +29,9 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@backstage/core": "^0.4.3", - "@backstage/catalog-model": "^0.6.0", - "@backstage/plugin-catalog": "^0.2.9", + "@backstage/core": "^0.5.0", + "@backstage/catalog-model": "^0.7.0", + "@backstage/plugin-catalog": "^0.2.12", "@backstage/test-utils": "^0.1.5", "@backstage/theme": "^0.2.2", "@material-ui/core": "^4.11.0", @@ -47,7 +47,7 @@ "react-router-dom": "6.0.0-beta.0" }, "devDependencies": { - "@backstage/cli": "^0.4.4", + "@backstage/cli": "^0.4.7", "@types/jest": "^26.0.7", "@types/node": "^12.0.0" }, diff --git a/packages/e2e-test/package.json b/packages/e2e-test/package.json index ea1dcc8ea2..4327d7161c 100644 --- a/packages/e2e-test/package.json +++ b/packages/e2e-test/package.json @@ -15,7 +15,7 @@ "license": "Apache-2.0", "main": "src/index.ts", "scripts": { - "start": "node .", + "start": "nodemon --", "lint": "backstage-cli lint", "test": "backstage-cli test", "test:e2e": "yarn start" @@ -36,5 +36,10 @@ "tree-kill": "^1.2.2", "ts-node": "^8.6.2", "zombie": "^6.1.4" + }, + "nodemonConfig": { + "watch": "./src", + "exec": "bin/e2e-test", + "ext": "ts" } } diff --git a/packages/e2e-test/src/commands/run.ts b/packages/e2e-test/src/commands/run.ts index be00b99e54..2ddcc333d2 100644 --- a/packages/e2e-test/src/commands/run.ts +++ b/packages/e2e-test/src/commands/run.ts @@ -80,10 +80,22 @@ async function buildDistWorkspace(workspaceName: string, rootDir: string) { // We grab the needed dependencies from the create app template const createAppDeps = new Set(); + + function appendDeps(pkg: any) { + Array() + .concat( + Object.keys(pkg.dependencies ?? {}), + Object.keys(pkg.devDependencies ?? {}), + Object.keys(pkg.peerDependencies ?? {}), + ) + .filter(name => name.startsWith('@backstage/')) + .forEach(dep => createAppDeps.add(dep)); + } + for (const pkgJsonPath of templatePackagePaths) { const path = paths.resolveOwnRoot(pkgJsonPath); const pkgTemplate = await fs.readFile(path, 'utf8'); - const { dependencies = {}, devDependencies = {} } = JSON.parse( + const pkg = JSON.parse( handlebars.compile(pkgTemplate)( { privatePackage: true, @@ -102,13 +114,12 @@ async function buildDistWorkspace(workspaceName: string, rootDir: string) { }, ), ); - - Array() - .concat(Object.keys(dependencies), Object.keys(devDependencies)) - .filter(name => name.startsWith('@backstage/')) - .forEach(dep => createAppDeps.add(dep)); + appendDeps(pkg); } + // eslint-disable-next-line import/no-extraneous-dependencies + appendDeps(require('@backstage/create-app/package.json')); + print(`Preparing workspace`); await runPlain([ 'yarn', diff --git a/packages/integration/CHANGELOG.md b/packages/integration/CHANGELOG.md index 2310c7d020..804fc0c63e 100644 --- a/packages/integration/CHANGELOG.md +++ b/packages/integration/CHANGELOG.md @@ -1,5 +1,80 @@ # @backstage/integration +## 0.3.0 + +### Minor Changes + +- ed6baab66: - Deprecating the `scaffolder.${provider}.token` auth duplication and favoring `integrations.${provider}` instead. If you receive deprecation warnings your config should change like the following: + + ```yaml + scaffolder: + github: + token: + $env: GITHUB_TOKEN + visibility: public + ``` + + To something that looks like this: + + ```yaml + integration: + github: + - host: github.com + token: + $env: GITHUB_TOKEN + scaffolder: + github: + visibility: public + ``` + + You can also configure multiple different hosts under the `integration` config like the following: + + ```yaml + integration: + github: + - host: github.com + token: + $env: GITHUB_TOKEN + - host: ghe.mycompany.com + token: + $env: GITHUB_ENTERPRISE_TOKEN + ``` + + This of course is the case for all the providers respectively. + + - Adding support for cross provider scaffolding, you can now create repositories in for example Bitbucket using a template residing in GitHub. + + - Fix GitLab scaffolding so that it returns a `catalogInfoUrl` which automatically imports the project into the catalog. + + - The `Store Path` field on the `scaffolder` frontend has now changed so that you require the full URL to the desired destination repository. + + `backstage/new-repository` would become `https://github.com/backstage/new-repository` if provider was GitHub for example. + +### Patch Changes + +- 0b135e7e0: Add support for GitHub Apps authentication for backend plugins. + + `GithubCredentialsProvider` requests and caches GitHub credentials based on a repository or organization url. + + The `GithubCredentialsProvider` class should be considered stateful since tokens will be cached internally. + Consecutive calls to get credentials will return the same token, tokens older than 50 minutes will be considered expired and reissued. + `GithubCredentialsProvider` will default to the configured access token if no GitHub Apps are configured. + + More information on how to create and configure a GitHub App to use with backstage can be found in the documentation. + + Usage: + + ```javascript + const credentialsProvider = new GithubCredentialsProvider(config); + const { token, headers } = await credentialsProvider.getCredentials({ + url: 'https://github.com/', + }); + ``` + + Updates `GithubUrlReader` to use the `GithubCredentialsProvider`. + +- fa8ba330a: Fix GitLab API base URL and add it by default to the gitlab.com host + ## 0.2.0 ### Minor Changes diff --git a/packages/integration/config.d.ts b/packages/integration/config.d.ts index 6a9fd62995..7d670ea505 100644 --- a/packages/integration/config.d.ts +++ b/packages/integration/config.d.ts @@ -15,41 +15,117 @@ */ export interface Config { + /** Configuration for integrations towards various external repository provider systems */ integrations?: { + /** Integration configuration for Azure */ azure?: Array<{ - /** @visibility frontend */ + /** + * The hostname of the given Azure instance + * @visibility frontend + */ host: string; + /** + * Token used to authenticate requests. + * @visibility secret + */ + token?: string; }>; + /** Integration configuration for Bitbucket */ bitbucket?: Array<{ - /** @visibility frontend */ + /** + * The hostname of the given Bitbucket instance + * @visibility frontend + */ host: string; - /** @visibility frontend */ + /** + * Token used to authenticate requests. + * @visibility secret + */ + token?: string; + /** + * The base url for the Bitbucket API, for example https://api.bitbucket.org/2.0 + * @visibility frontend + */ apiBaseUrl?: string; + /** + * The username to use for authenticated requests. + * @visibility secret + */ + username?: string; + /** + * Bitbucket app password used to authenticate requests. + * @visibility secret + */ + appPassword?: string; }>; + /** Integration configuration for GitHub */ github?: Array<{ - /** @visibility frontend */ + /** + * The hostname of the given GitHub instance + * @visibility frontend + */ host: string; - /** @visibility frontend */ + /** + * Token used to authenticate requests. + * @visibility secret + */ + token?: string; + /** + * The base url for the GitHub API, for example https://api.github.com + * @visibility frontend + */ apiBaseUrl?: string; - /** @visibility frontend */ + /** + * The base url for GitHub raw resources, for example https://raw.githubusercontent.com + * @visibility frontend + */ rawBaseUrl?: string; + + /** + * GitHub Apps configuration + * @visibility backend + */ apps?: Array<{ + /** + * The numeric GitHub App ID + */ appId: number; - /** @visiblity secret */ + /** + * The private key to use for auth against the app + * @visibility secret + */ privateKey: string; - /** @visiblity secret */ + /** + * The secret used for webhooks + * @visibility secret + */ webhookSecret: string; + /** + * The client ID to use + */ clientId: string; - /** @visiblity secret */ + /** + * The client secret to use + * @visibility secret + */ clientSecret: string; }>; }>; + /** Integration configuration for GitLab */ gitlab?: Array<{ - /** @visibility frontend */ + /** + * The hostname of the given GitLab instance + * @visibility frontend + */ host: string; + /** + * Token used to authenticate requests. + * @visibility secret + */ + token?: string; }>; }; } diff --git a/packages/integration/package.json b/packages/integration/package.json index 8a8804c8e5..fd5268dced 100644 --- a/packages/integration/package.json +++ b/packages/integration/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/integration", - "version": "0.2.0", + "version": "0.3.0", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -37,7 +37,7 @@ "luxon": "^1.25.0" }, "devDependencies": { - "@backstage/cli": "^0.4.6", + "@backstage/cli": "^0.4.7", "@backstage/test-utils": "^0.1.5", "@types/jest": "^26.0.7", "@types/luxon": "^1.25.0", diff --git a/packages/integration/src/azure/core.ts b/packages/integration/src/azure/core.ts index 7900774c79..b2878af89e 100644 --- a/packages/integration/src/azure/core.ts +++ b/packages/integration/src/azure/core.ts @@ -108,6 +108,62 @@ export function getAzureDownloadUrl(url: string): string { return `${protocol}://${resource}/${organization}/${project}/_apis/git/repositories/${repoName}/items?recursionLevel=full&download=true&api-version=6.0${scopePath}`; } +/** + * Given a URL, return the API URL to fetch commits on the branch. + * + * @param url A URL pointing to a repository or a sub-path + */ +export function getAzureCommitsUrl(url: string): string { + try { + const parsedUrl = new URL(url); + + const [ + empty, + userOrOrg, + project, + srcKeyword, + repoName, + ] = parsedUrl.pathname.split('/'); + + // Remove the "GB" from "GBmain" for example. + const ref = parsedUrl.searchParams.get('version')?.substr(2); + + if ( + !!empty || + !userOrOrg || + !project || + srcKeyword !== '_git' || + !repoName + ) { + throw new Error('Wrong Azure Devops URL'); + } + + // transform to commits api + parsedUrl.pathname = [ + empty, + userOrOrg, + project, + '_apis', + 'git', + 'repositories', + repoName, + 'commits', + ].join('/'); + + const queryParams = []; + if (ref) { + queryParams.push(`searchCriteria.itemVersion.version=${ref}`); + } + parsedUrl.search = queryParams.join('&'); + + parsedUrl.protocol = 'https'; + + return parsedUrl.toString(); + } catch (e) { + throw new Error(`Incorrect URL: ${url}, ${e}`); + } +} + /** * Gets the request options necessary to make requests to a given provider. * diff --git a/packages/integration/src/azure/index.ts b/packages/integration/src/azure/index.ts index 365e4cdcdc..6d57437779 100644 --- a/packages/integration/src/azure/index.ts +++ b/packages/integration/src/azure/index.ts @@ -23,4 +23,5 @@ export { getAzureDownloadUrl, getAzureFileFetchUrl, getAzureRequestOptions, + getAzureCommitsUrl, } from './core'; diff --git a/packages/integration/src/github/GithubCredentialsProvider.ts b/packages/integration/src/github/GithubCredentialsProvider.ts index 843ef629ab..dfb809ee5f 100644 --- a/packages/integration/src/github/GithubCredentialsProvider.ts +++ b/packages/integration/src/github/GithubCredentialsProvider.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import gitUrlParse from 'git-url-parse'; +import parseGitUrl from 'git-url-parse'; import { GithubAppConfig, GitHubIntegrationConfig } from './config'; import { createAppAuth } from '@octokit/auth-app'; import { Octokit, RestEndpointMethodTypes } from '@octokit/rest'; @@ -221,7 +221,7 @@ export class GithubCredentialsProvider { * const { token, headers } = await getCredentials({url: 'github.com/backstage/foobar'}) */ async getCredentials(opts: { url: string }): Promise { - const parsed = gitUrlParse(opts.url); + const parsed = parseGitUrl(opts.url); const owner = parsed.owner || parsed.name; const repo = parsed.owner ? parsed.name : undefined; diff --git a/packages/integration/src/gitlab/config.test.ts b/packages/integration/src/gitlab/config.test.ts index e817465b83..303b4bd270 100644 --- a/packages/integration/src/gitlab/config.test.ts +++ b/packages/integration/src/gitlab/config.test.ts @@ -31,11 +31,14 @@ describe('readGitLabIntegrationConfig', () => { buildConfig({ host: 'a.com', token: 't', + baseUrl: 'https://baseurl.for.me/gitlab', }), ); + expect(output).toEqual({ host: 'a.com', token: 't', + baseUrl: 'https://baseurl.for.me/gitlab', }); }); @@ -43,7 +46,20 @@ describe('readGitLabIntegrationConfig', () => { const output = readGitLabIntegrationConfig(buildConfig({})); expect(output).toEqual({ host: 'gitlab.com', - apiBaseUrl: 'gitlab.com/api/v4', + apiBaseUrl: 'https://gitlab.com/api/v4', + baseUrl: 'https://gitlab.com', + }); + }); + + it('injects the correct GitLab API base URL when missing', () => { + const output = readGitLabIntegrationConfig( + buildConfig({ host: 'gitlab.com' }), + ); + + expect(output).toEqual({ + host: 'gitlab.com', + baseUrl: 'https://gitlab.com', + apiBaseUrl: 'https://gitlab.com/api/v4', }); }); @@ -78,6 +94,7 @@ describe('readGitLabIntegrationConfigs', () => { expect(output).toContainEqual({ host: 'a.com', token: 't', + baseUrl: 'https://a.com', }); }); @@ -86,6 +103,7 @@ describe('readGitLabIntegrationConfigs', () => { expect(output).toEqual([ { host: 'gitlab.com', + apiBaseUrl: 'https://gitlab.com/api/v4', }, ]); }); diff --git a/packages/integration/src/gitlab/config.ts b/packages/integration/src/gitlab/config.ts index 2d9f4b39ab..47269b4c32 100644 --- a/packages/integration/src/gitlab/config.ts +++ b/packages/integration/src/gitlab/config.ts @@ -18,7 +18,7 @@ import { Config } from '@backstage/config'; import { isValidHost } from '../helpers'; const GITLAB_HOST = 'gitlab.com'; -const GITLAB_API_BASE_URL = 'gitlab.com/api/v4'; +const GITLAB_API_BASE_URL = 'https://gitlab.com/api/v4'; /** * The configuration parameters for a single GitLab integration. @@ -46,6 +46,14 @@ export type GitLabIntegrationConfig = { * If no token is specified, anonymous access is used. */ token?: string; + + /** + * The baseUrl of this provider, e.g "https://gitlab.com", + * which is passed into the gitlab client. + * + * If no baseUrl is provided, it will default to https://${host} + */ + baseUrl?: string; }; /** @@ -59,6 +67,7 @@ export function readGitLabIntegrationConfig( const host = config.getOptionalString('host') ?? GITLAB_HOST; let apiBaseUrl = config.getOptionalString('apiBaseUrl'); const token = config.getOptionalString('token'); + const baseUrl = config.getOptionalString('baseUrl') ?? `https://${host}`; if (!isValidHost(host)) { throw new Error( @@ -71,7 +80,8 @@ export function readGitLabIntegrationConfig( } else if (host === GITLAB_HOST) { apiBaseUrl = GITLAB_API_BASE_URL; } - return { host, token, apiBaseUrl }; + + return { host, token, apiBaseUrl, baseUrl }; } /** @@ -89,7 +99,7 @@ export function readGitLabIntegrationConfigs( // As a convenience we always make sure there's at least an unauthenticated // reader for public gitlab repos. if (!result.some(c => c.host === GITLAB_HOST)) { - result.push({ host: GITLAB_HOST }); + result.push({ host: GITLAB_HOST, apiBaseUrl: GITLAB_API_BASE_URL }); } return result; diff --git a/packages/techdocs-common/CHANGELOG.md b/packages/techdocs-common/CHANGELOG.md index c26164bbb6..7bff39d3c4 100644 --- a/packages/techdocs-common/CHANGELOG.md +++ b/packages/techdocs-common/CHANGELOG.md @@ -1,5 +1,27 @@ # @backstage/techdocs-common +## 0.3.5 + +### Patch Changes + +- 53c9c51f2: TechDocs backend now streams files through from Google Cloud Storage to the browser, improving memory usage. +- a5e27d5c1: Create type for TechDocsMetadata (#3716) + + This change introduces a new type (TechDocsMetadata) in packages/techdocs-common. This type is then introduced in the endpoint response in techdocs-backend and in the api interface in techdocs (frontend). + +- Updated dependencies [def2307f3] +- Updated dependencies [0b135e7e0] +- Updated dependencies [294a70cab] +- Updated dependencies [fa8ba330a] +- Updated dependencies [0ea032763] +- Updated dependencies [5345a1f98] +- Updated dependencies [ed6baab66] +- Updated dependencies [09a370426] +- Updated dependencies [a93f42213] + - @backstage/catalog-model@0.7.0 + - @backstage/backend-common@0.5.0 + - @backstage/integration@0.3.0 + ## 0.3.4 ### Patch Changes diff --git a/packages/techdocs-common/package.json b/packages/techdocs-common/package.json index a069326375..0a1a673f3a 100644 --- a/packages/techdocs-common/package.json +++ b/packages/techdocs-common/package.json @@ -1,7 +1,7 @@ { "name": "@backstage/techdocs-common", "description": "Common functionalities for TechDocs, to be shared between techdocs-backend plugin and techdocs-cli", - "version": "0.3.4", + "version": "0.3.5", "main": "src/index.ts", "types": "src/index.ts", "private": false, @@ -37,10 +37,10 @@ }, "dependencies": { "@aws-sdk/client-s3": "^3.1.0", - "@backstage/backend-common": "^0.4.3", - "@backstage/catalog-model": "^0.6.1", + "@backstage/backend-common": "^0.5.0", + "@backstage/catalog-model": "^0.7.0", "@backstage/config": "^0.1.2", - "@backstage/integration": "^0.2.0", + "@backstage/integration": "^0.3.0", "@google-cloud/storage": "^5.6.0", "@types/dockerode": "^3.2.1", "@types/express": "^4.17.6", @@ -50,6 +50,7 @@ "fs-extra": "^9.0.1", "git-url-parse": "^11.4.3", "js-yaml": "^4.0.0", + "json5": "^2.1.3", "mime-types": "^2.1.27", "mock-fs": "^4.13.0", "recursive-readdir": "^2.2.2", @@ -57,7 +58,7 @@ }, "devDependencies": { "@aws-sdk/types": "3.1.0", - "@backstage/cli": "^0.4.6", + "@backstage/cli": "^0.4.7", "@types/fs-extra": "^9.0.5", "@types/git-url-parse": "^9.0.0", "@types/js-yaml": "^3.12.5", diff --git a/packages/techdocs-common/src/helpers.test.ts b/packages/techdocs-common/src/helpers.test.ts index 88c4d8829e..740a08aaa2 100644 --- a/packages/techdocs-common/src/helpers.test.ts +++ b/packages/techdocs-common/src/helpers.test.ts @@ -135,6 +135,7 @@ describe('getDocFilesFromRepository', () => { archive: async () => { return Readable.from(''); }, + etag: '', }; } } diff --git a/packages/techdocs-common/src/stages/publish/awsS3.test.ts b/packages/techdocs-common/src/stages/publish/awsS3.test.ts index c157a7c205..3902aab21a 100755 --- a/packages/techdocs-common/src/stages/publish/awsS3.test.ts +++ b/packages/techdocs-common/src/stages/publish/awsS3.test.ts @@ -18,7 +18,7 @@ import path from 'path'; import * as winston from 'winston'; import { ConfigReader } from '@backstage/config'; import { AwsS3Publish } from './awsS3'; -import { PublisherBase } from './types'; +import { PublisherBase, TechDocsMetadata } from './types'; import type { Entity, EntityName } from '@backstage/catalog-model'; const createMockEntity = (annotations = {}): Entity => { @@ -159,13 +159,39 @@ describe('AwsS3Publish', () => { mockFs({ [entityRootDir]: { - 'techdocs_metadata.json': 'file-content', + 'techdocs_metadata.json': + '{"site_name": "backstage", "site_description": "site_content"}', }, }); - expect(await publisher.fetchTechDocsMetadata(entityNameMock)).toBe( - 'file-content', - ); + const expectedMetadata: TechDocsMetadata = { + site_name: 'backstage', + site_description: 'site_content', + }; + expect( + await publisher.fetchTechDocsMetadata(entityNameMock), + ).toStrictEqual(expectedMetadata); + mockFs.restore(); + }); + + it('should return tech docs metadata when json encoded with single quotes', async () => { + const entityNameMock = createMockEntityName(); + const entity = createMockEntity(); + const entityRootDir = getEntityRootDir(entity); + + mockFs({ + [entityRootDir]: { + 'techdocs_metadata.json': `{'site_name': 'backstage', 'site_description': 'site_content'}`, + }, + }); + + const expectedMetadata: TechDocsMetadata = { + site_name: 'backstage', + site_description: 'site_content', + }; + expect( + await publisher.fetchTechDocsMetadata(entityNameMock), + ).toStrictEqual(expectedMetadata); mockFs.restore(); }); diff --git a/packages/techdocs-common/src/stages/publish/awsS3.ts b/packages/techdocs-common/src/stages/publish/awsS3.ts index 3f7a0e3d81..b9e9fe12e6 100644 --- a/packages/techdocs-common/src/stages/publish/awsS3.ts +++ b/packages/techdocs-common/src/stages/publish/awsS3.ts @@ -20,9 +20,10 @@ import { Logger } from 'winston'; import { Entity, EntityName } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; import { getHeadersForFileExtension, getFileTreeRecursively } from './helpers'; -import { PublisherBase, PublishRequest } from './types'; +import { PublisherBase, PublishRequest, TechDocsMetadata } from './types'; import fs from 'fs-extra'; import { Readable } from 'stream'; +import JSON5 from 'json5'; const streamToBuffer = (stream: Readable): Promise => { return new Promise((resolve, reject) => { @@ -32,7 +33,7 @@ const streamToBuffer = (stream: Readable): Promise => { stream.on('error', reject); stream.on('end', () => resolve(Buffer.concat(chunks))); } catch (e) { - throw new Error(`Unable to parse the response data, ${e.message}`); + throw new Error(`Unable to parse the response data ${e.message}`); } }); }; @@ -162,9 +163,11 @@ export class AwsS3Publish implements PublisherBase { } } - async fetchTechDocsMetadata(entityName: EntityName): Promise { + async fetchTechDocsMetadata( + entityName: EntityName, + ): Promise { try { - return await new Promise((resolve, reject) => { + return await new Promise((resolve, reject) => { const entityRootDir = `${entityName.namespace}/${entityName.kind}/${entityName.name}`; this.storageClient @@ -182,8 +185,11 @@ export class AwsS3Publish implements PublisherBase { `Unable to parse the techdocs metadata file ${entityRootDir}/techdocs_metadata.json.`, ); } + const techdocsMetadata = JSON5.parse( + techdocsMetadataJson.toString('utf-8'), + ); - resolve(techdocsMetadataJson.toString('utf-8')); + resolve(techdocsMetadata); }) .catch(err => { this.logger.error(err.message); diff --git a/packages/techdocs-common/src/stages/publish/googleStorage.ts b/packages/techdocs-common/src/stages/publish/googleStorage.ts index 9c28de4130..82a9620e8f 100644 --- a/packages/techdocs-common/src/stages/publish/googleStorage.ts +++ b/packages/techdocs-common/src/stages/publish/googleStorage.ts @@ -24,7 +24,8 @@ import { Logger } from 'winston'; import { Entity, EntityName } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; import { getHeadersForFileExtension, getFileTreeRecursively } from './helpers'; -import { PublisherBase, PublishRequest } from './types'; +import { PublisherBase, PublishRequest, TechDocsMetadata } from './types'; +import JSON5 from 'json5'; export class GoogleGCSPublish implements PublisherBase { static async fromConfig( @@ -132,7 +133,7 @@ export class GoogleGCSPublish implements PublisherBase { }); } - fetchTechDocsMetadata(entityName: EntityName): Promise { + fetchTechDocsMetadata(entityName: EntityName): Promise { return new Promise((resolve, reject) => { const entityRootDir = `${entityName.namespace}/${entityName.kind}/${entityName.name}`; @@ -152,7 +153,7 @@ export class GoogleGCSPublish implements PublisherBase { const techdocsMetadataJson = Buffer.concat( fileStreamChunks, ).toString(); - resolve(techdocsMetadataJson); + resolve(JSON5.parse(techdocsMetadataJson)); }); }); } diff --git a/packages/techdocs-common/src/stages/publish/index.ts b/packages/techdocs-common/src/stages/publish/index.ts index 494efe5ac1..dbf4ff732a 100644 --- a/packages/techdocs-common/src/stages/publish/index.ts +++ b/packages/techdocs-common/src/stages/publish/index.ts @@ -14,4 +14,4 @@ * limitations under the License. */ export { Publisher } from './publish'; -export type { PublisherBase, PublisherType } from './types'; +export type { PublisherBase, PublisherType, TechDocsMetadata } from './types'; diff --git a/packages/techdocs-common/src/stages/publish/local.ts b/packages/techdocs-common/src/stages/publish/local.ts index de670cfba5..d07472e17d 100644 --- a/packages/techdocs-common/src/stages/publish/local.ts +++ b/packages/techdocs-common/src/stages/publish/local.ts @@ -25,7 +25,12 @@ import { PluginEndpointDiscovery, } from '@backstage/backend-common'; import { Config } from '@backstage/config'; -import { PublisherBase, PublishRequest, PublishResponse } from './types'; +import { + PublisherBase, + PublishRequest, + PublishResponse, + TechDocsMetadata, +} from './types'; // TODO: Use a more persistent storage than node_modules or /tmp directory. // Make it configurable with techdocs.publisher.local.publishDirectory @@ -102,7 +107,7 @@ export class LocalPublish implements PublisherBase { }); } - fetchTechDocsMetadata(entityName: EntityName): Promise { + fetchTechDocsMetadata(entityName: EntityName): Promise { return new Promise((resolve, reject) => { this.discovery.getBaseUrl('techdocs').then(techdocsApiUrl => { const storageUrl = new URL( @@ -116,7 +121,7 @@ export class LocalPublish implements PublisherBase { .then(response => response .json() - .then(techdocsMetadataJson => resolve(techdocsMetadataJson)) + .then(techdocsMetadata => resolve(techdocsMetadata)) .catch(err => { reject( `Unable to parse metadata JSON for ${entityRootDir}. Error: ${err}`, diff --git a/packages/techdocs-common/src/stages/publish/types.ts b/packages/techdocs-common/src/stages/publish/types.ts index 24a9d05adf..db6a075d43 100644 --- a/packages/techdocs-common/src/stages/publish/types.ts +++ b/packages/techdocs-common/src/stages/publish/types.ts @@ -32,6 +32,14 @@ export type PublishResponse = { remoteUrl?: string; } | void; +/** + * Type to hold metadata found in techdocs_metadata.json and associated with each site + */ +export type TechDocsMetadata = { + site_name: string; + site_description: string; +}; + /** * Base class for a TechDocs publisher (e.g. Local, Google GCS Bucket, AWS S3, etc.) * The publisher handles publishing of the generated static files after the prepare and generate steps of TechDocs. @@ -50,7 +58,7 @@ export interface PublisherBase { * Retrieve TechDocs Metadata about a site e.g. name, contributors, last updated, etc. * This API uses the techdocs_metadata.json file that co-exists along with the generated docs. */ - fetchTechDocsMetadata(entityName: EntityName): Promise; + fetchTechDocsMetadata(entityName: EntityName): Promise; /** * Route middleware to serve static documentation files for an entity. diff --git a/plugins/api-docs/CHANGELOG.md b/plugins/api-docs/CHANGELOG.md index 995dfdc175..01e34d8e1d 100644 --- a/plugins/api-docs/CHANGELOG.md +++ b/plugins/api-docs/CHANGELOG.md @@ -1,5 +1,21 @@ # @backstage/plugin-api-docs +## 0.4.3 + +### Patch Changes + +- 8855f61f6: Update `@asyncapi/react-component` to 0.18.2 +- Updated dependencies [def2307f3] +- Updated dependencies [efd6ef753] +- Updated dependencies [593632f07] +- Updated dependencies [33846acfc] +- Updated dependencies [a187b8ad0] +- Updated dependencies [f04db53d7] +- Updated dependencies [a93f42213] + - @backstage/catalog-model@0.7.0 + - @backstage/core@0.5.0 + - @backstage/plugin-catalog@0.2.12 + ## 0.4.2 ### Patch Changes diff --git a/plugins/api-docs/package.json b/plugins/api-docs/package.json index 39e202eef8..d9c0dc0968 100644 --- a/plugins/api-docs/package.json +++ b/plugins/api-docs/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-api-docs", - "version": "0.4.2", + "version": "0.4.3", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -29,11 +29,11 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@backstage/catalog-model": "^0.6.0", - "@backstage/core": "^0.4.4", - "@backstage/plugin-catalog": "^0.2.9", + "@asyncapi/react-component": "^0.18.2", + "@backstage/catalog-model": "^0.7.0", + "@backstage/core": "^0.5.0", + "@backstage/plugin-catalog": "^0.2.12", "@backstage/theme": "^0.2.2", - "@kyma-project/asyncapi-react": "^0.14.2", "@material-icons/font": "^1.0.2", "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", @@ -49,8 +49,8 @@ "swagger-ui-react": "^3.37.2" }, "devDependencies": { - "@backstage/cli": "^0.4.6", - "@backstage/dev-utils": "^0.1.7", + "@backstage/cli": "^0.4.7", + "@backstage/dev-utils": "^0.1.8", "@backstage/test-utils": "^0.1.6", "@testing-library/jest-dom": "^5.10.1", "@testing-library/react": "^10.4.1", diff --git a/plugins/api-docs/src/components/AsyncApiDefinitionWidget/AsyncApiDefinitionWidget.tsx b/plugins/api-docs/src/components/AsyncApiDefinitionWidget/AsyncApiDefinitionWidget.tsx index 86f524d099..a908a91b14 100644 --- a/plugins/api-docs/src/components/AsyncApiDefinitionWidget/AsyncApiDefinitionWidget.tsx +++ b/plugins/api-docs/src/components/AsyncApiDefinitionWidget/AsyncApiDefinitionWidget.tsx @@ -14,10 +14,10 @@ * limitations under the License. */ -import AsyncApi from '@kyma-project/asyncapi-react'; +import AsyncApi from '@asyncapi/react-component'; +import '@asyncapi/react-component/lib/styles/fiori.css'; +import { fade, makeStyles } from '@material-ui/core/styles'; import React from 'react'; -import { makeStyles, fade } from '@material-ui/core/styles'; -import '@kyma-project/asyncapi-react/lib/styles/fiori.css'; const useStyles = makeStyles(theme => ({ root: { diff --git a/plugins/app-backend/CHANGELOG.md b/plugins/app-backend/CHANGELOG.md index e16ac1fb9c..90a4eac95d 100644 --- a/plugins/app-backend/CHANGELOG.md +++ b/plugins/app-backend/CHANGELOG.md @@ -1,5 +1,16 @@ # @backstage/plugin-app-backend +## 0.3.4 + +### Patch Changes + +- Updated dependencies [0b135e7e0] +- Updated dependencies [294a70cab] +- Updated dependencies [0ea032763] +- Updated dependencies [5345a1f98] +- Updated dependencies [09a370426] + - @backstage/backend-common@0.5.0 + ## 0.3.3 ### Patch Changes diff --git a/plugins/app-backend/package.json b/plugins/app-backend/package.json index 0dde9c9d13..f55da9042a 100644 --- a/plugins/app-backend/package.json +++ b/plugins/app-backend/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-app-backend", - "version": "0.3.3", + "version": "0.3.4", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -29,7 +29,7 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@backstage/backend-common": "^0.4.0", + "@backstage/backend-common": "^0.5.0", "@backstage/config-loader": "^0.4.0", "@backstage/config": "^0.1.2", "@types/express": "^4.17.6", @@ -40,7 +40,7 @@ "yn": "^4.0.0" }, "devDependencies": { - "@backstage/cli": "^0.4.1", + "@backstage/cli": "^0.4.7", "@types/supertest": "^2.0.8", "msw": "^0.20.5", "supertest": "^4.0.2" diff --git a/plugins/auth-backend/CHANGELOG.md b/plugins/auth-backend/CHANGELOG.md index f21224eb65..0df0fa1938 100644 --- a/plugins/auth-backend/CHANGELOG.md +++ b/plugins/auth-backend/CHANGELOG.md @@ -1,5 +1,22 @@ # @backstage/plugin-auth-backend +## 0.2.11 + +### Patch Changes + +- 0643a3336: Add AWS ALB OIDC reverse proxy authentication provider +- a2291d7cc: Optional identity token authorization of api requests +- Updated dependencies [def2307f3] +- Updated dependencies [0b135e7e0] +- Updated dependencies [294a70cab] +- Updated dependencies [0ea032763] +- Updated dependencies [5345a1f98] +- Updated dependencies [09a370426] +- Updated dependencies [a93f42213] + - @backstage/catalog-model@0.7.0 + - @backstage/backend-common@0.5.0 + - @backstage/catalog-client@0.3.5 + ## 0.2.10 ### Patch Changes diff --git a/plugins/auth-backend/package.json b/plugins/auth-backend/package.json index 8f88c69487..27cb725faa 100644 --- a/plugins/auth-backend/package.json +++ b/plugins/auth-backend/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-auth-backend", - "version": "0.2.10", + "version": "0.2.11", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -29,9 +29,9 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@backstage/backend-common": "^0.4.3", - "@backstage/catalog-client": "^0.3.4", - "@backstage/catalog-model": "^0.6.1", + "@backstage/backend-common": "^0.5.0", + "@backstage/catalog-client": "^0.3.5", + "@backstage/catalog-model": "^0.7.0", "@backstage/config": "^0.1.2", "@types/express": "^4.17.6", "compression": "^1.7.4", @@ -49,6 +49,7 @@ "knex": "^0.21.6", "moment": "^2.26.0", "morgan": "^1.10.0", + "node-cache": "^5.1.2", "openid-client": "^4.2.1", "passport": "^0.4.1", "passport-github2": "^0.1.12", @@ -64,7 +65,7 @@ "yn": "^4.0.0" }, "devDependencies": { - "@backstage/cli": "^0.4.6", + "@backstage/cli": "^0.4.7", "@types/body-parser": "^1.19.0", "@types/cookie-parser": "^1.4.2", "@types/express-session": "^1.17.2", diff --git a/plugins/auth-backend/src/identity/IdentityClient.test.ts b/plugins/auth-backend/src/identity/IdentityClient.test.ts new file mode 100644 index 0000000000..5f317aa0c2 --- /dev/null +++ b/plugins/auth-backend/src/identity/IdentityClient.test.ts @@ -0,0 +1,248 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { JWT, JSONWebKey } from 'jose'; +import { rest } from 'msw'; +import { setupServer } from 'msw/node'; +import { + getVoidLogger, + PluginEndpointDiscovery, +} from '@backstage/backend-common'; +import { IdentityClient } from './IdentityClient'; +import { MemoryKeyStore } from './MemoryKeyStore'; +import { TokenFactory } from './TokenFactory'; +import { KeyStore } from './types'; + +const logger = getVoidLogger(); + +function jwtKid(jwt: string): string { + const { header } = JWT.decode(jwt, { complete: true }) as { + header: { kid: string }; + }; + return header.kid; +} + +const server = setupServer(); +const mockBaseUrl = 'http://backstage:9191/i-am-a-mock-base'; +const discovery: PluginEndpointDiscovery = { + async getBaseUrl() { + return mockBaseUrl; + }, + async getExternalBaseUrl() { + return mockBaseUrl; + }, +}; + +describe('IdentityClient', () => { + let client: IdentityClient; + let factory: TokenFactory; + let keyStore: KeyStore; + const keyDurationSeconds = 5; + + beforeAll(() => server.listen({ onUnhandledRequest: 'error' })); + afterAll(() => server.close()); + afterEach(() => server.resetHandlers()); + + beforeEach(() => { + client = new IdentityClient({ discovery, issuer: mockBaseUrl }); + keyStore = new MemoryKeyStore(); + factory = new TokenFactory({ + issuer: mockBaseUrl, + keyStore: keyStore, + keyDurationSeconds, + logger, + }); + }); + + describe('authenticate', () => { + beforeEach(() => { + server.use( + rest.get( + `${mockBaseUrl}/.well-known/jwks.json`, + async (_, res, ctx) => { + const keys = await factory.listPublicKeys(); + return res(ctx.json(keys)); + }, + ), + ); + }); + + it('should use the correct endpoint', async () => { + await factory.issueToken({ claims: { sub: 'foo' } }); + const keys = await factory.listPublicKeys(); + const response = await client.listPublicKeys(); + expect(response).toEqual(keys); + }); + + it('should throw on undefined header', async () => { + return expect(async () => { + await client.authenticate(undefined); + }).rejects.toThrow(); + }); + + it('should accept fresh token', async () => { + const token = await factory.issueToken({ claims: { sub: 'foo' } }); + const response = await client.authenticate(token); + expect(response).toEqual({ id: 'foo', idToken: token }); + }); + + it('should throw on incorrect issuer', async () => { + const hackerFactory = new TokenFactory({ + issuer: 'hacker', + keyStore, + keyDurationSeconds, + logger, + }); + return expect(async () => { + const token = await hackerFactory.issueToken({ + claims: { sub: 'foo' }, + }); + await client.authenticate(token); + }).rejects.toThrow(); + }); + + it('should throw on expired token', async () => { + return expect(async () => { + const fixedTime = Date.now(); + jest + .spyOn(Date, 'now') + .mockImplementation(() => fixedTime - keyDurationSeconds * 1000 * 2); + const token = await factory.issueToken({ + claims: { sub: 'foo' }, + }); + jest.spyOn(Date, 'now').mockImplementation(() => fixedTime); + await client.authenticate(token); + }).rejects.toThrow(); + }); + + it('should throw on incorrect signing key', async () => { + const hackerFactory = new TokenFactory({ + issuer: mockBaseUrl, + keyStore: new MemoryKeyStore(), + keyDurationSeconds, + logger, + }); + return expect(async () => { + const token = await hackerFactory.issueToken({ + claims: { sub: 'foo' }, + }); + await client.authenticate(token); + }).rejects.toThrow(); + }); + + it('should accept token from new key', async () => { + const fixedTime = Date.now(); + jest + .spyOn(Date, 'now') + .mockImplementation(() => fixedTime - keyDurationSeconds * 1000 * 2); + const token1 = await factory.issueToken({ claims: { sub: 'foo1' } }); + try { + // This throws as token has already expired + await client.authenticate(token1); + } catch (_err) { + // Ignore thrown error + } + // Move forward in time where the signing key has been rotated + jest.spyOn(Date, 'now').mockImplementation(() => fixedTime); + const token = await factory.issueToken({ claims: { sub: 'foo' } }); + const response = await client.authenticate(token); + expect(response).toEqual({ id: 'foo', idToken: token }); + }); + + it('should not be fooled by the none algorithm', async () => { + return expect(async () => { + const token = await factory.issueToken({ claims: { sub: 'foo' } }); + const header = btoa( + JSON.stringify({ alg: 'none', kid: jwtKid(token) }), + ); + const payload = btoa( + JSON.stringify({ + iss: mockBaseUrl, + sub: 'foo', + aud: 'backstage', + iat: Date.now() / 1000, + exp: Date.now() / 1000 + 60000, + }), + ); + const fakeToken = `${header}.${payload}.`; + return await client.authenticate(fakeToken); + }).rejects.toThrow(); + }); + }); + + describe('getBearerToken', () => { + it('should return undefined on undefined input', async () => { + const token = IdentityClient.getBearerToken(undefined); + expect(token).toBeUndefined(); + }); + + it('should return undefined on malformed input', async () => { + const token = IdentityClient.getBearerToken('malformed'); + expect(token).toBeUndefined(); + }); + + it('should return undefined on unexpected scheme', async () => { + const token = IdentityClient.getBearerToken('Basic token'); + expect(token).toBeUndefined(); + }); + + it('should return Bearer token', async () => { + const token = IdentityClient.getBearerToken('Bearer token'); + expect(token).toEqual('token'); + }); + + it('should return Bearer token despite extra space', async () => { + const token = IdentityClient.getBearerToken('Bearer \n token '); + expect(token).toEqual('token'); + }); + + it('should return Bearer token despite unconventionial case', async () => { + const token = IdentityClient.getBearerToken('bEARER token'); + expect(token).toEqual('token'); + }); + }); + + describe('listPublicKeys', () => { + const defaultServiceResponse: { + keys: JSONWebKey[]; + } = { + keys: [ + { + crv: 'P-256', + x: 'JWy80Goa-8C3oaeDLnk0ANVPPMfI9T3u_T5T7W2b_ls', + y: 'Ge6jAhCDW1PFBfme2RA5ZsXN0cESiCwW29LMRPX5wkw', + kty: 'EC', + kid: 'kid-a', + alg: 'ES256', + use: 'sig', + }, + ], + }; + + beforeEach(() => { + server.use( + rest.get(`${mockBaseUrl}/.well-known/jwks.json`, (_, res, ctx) => { + return res(ctx.json(defaultServiceResponse)); + }), + ); + }); + + it('should use the correct endpoint', async () => { + const response = await client.listPublicKeys(); + expect(response).toEqual(defaultServiceResponse); + }); + }); +}); diff --git a/plugins/auth-backend/src/identity/IdentityClient.ts b/plugins/auth-backend/src/identity/IdentityClient.ts new file mode 100644 index 0000000000..970c30dd40 --- /dev/null +++ b/plugins/auth-backend/src/identity/IdentityClient.ts @@ -0,0 +1,146 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import fetch from 'cross-fetch'; +import { JWK, JWT, JWKS, JSONWebKey } from 'jose'; +import { BackstageIdentity } from '../providers'; +import { PluginEndpointDiscovery } from '@backstage/backend-common'; + +const CLOCK_MARGIN_S = 10; + +/** + * A identity client to interact with auth-backend + * and authenticate backstage identity tokens + * + * @experimental This is not a stable API yet + */ +export class IdentityClient { + private readonly discovery: PluginEndpointDiscovery; + private readonly issuer: string; + private keyStore: JWKS.KeyStore; + private keyStoreUpdated: number; + + constructor(options: { discovery: PluginEndpointDiscovery; issuer: string }) { + this.discovery = options.discovery; + this.issuer = options.issuer; + this.keyStore = new JWKS.KeyStore(); + this.keyStoreUpdated = 0; + } + + /** + * Verifies the given backstage identity token + * Returns a BackstageIdentity (user) matching the token. + * The method throws an error if verification fails. + */ + async authenticate(token: string | undefined): Promise { + // Extract token from header + if (!token) { + throw new Error('No token specified'); + } + // Get signing key matching token + const key = await this.getKey(token); + if (!key) { + throw new Error('No signing key matching token found'); + } + // Verify token claims and signature + // Note: Claims must match those set by TokenFactory when issuing tokens + // Note: verify throws if verification fails + const decoded = JWT.IdToken.verify(token, key, { + algorithms: ['ES256'], + audience: 'backstage', + issuer: this.issuer, + }) as { sub: string }; + // Verified, return the matching user as BackstageIdentity + // TODO: Settle internal user format/properties + const user: BackstageIdentity = { + id: decoded.sub, + idToken: token, + }; + return user; + } + + /** + * Parses the given authorization header and returns + * the bearer token, or null if no bearer token is given + */ + static getBearerToken( + authorizationHeader: string | undefined, + ): string | undefined { + if (typeof authorizationHeader !== 'string') { + return undefined; + } + const matches = authorizationHeader.match(/Bearer\s+(\S+)/i); + return matches?.[1]; + } + + /** + * Returns the public signing key matching the given jwt token, + * or null if no matching key was found + */ + private async getKey(rawJwtToken: string): Promise { + const { header, payload } = JWT.decode(rawJwtToken, { + complete: true, + }) as { + header: { kid: string }; + payload: { iat: number }; + }; + + // Refresh public keys if needed + // Add a small margin in case clocks are out of sync + const keyStoreHasKey = !!this.keyStore.get({ kid: header.kid }); + const issuedAfterLastRefresh = + payload?.iat && payload.iat > this.keyStoreUpdated - CLOCK_MARGIN_S; + if (!keyStoreHasKey && issuedAfterLastRefresh) { + await this.refreshKeyStore(); + } + + return this.keyStore.get({ kid: header.kid }); + } + + /** + * Lists public part of keys used to sign Backstage Identity tokens + */ + async listPublicKeys(): Promise<{ + keys: JSONWebKey[]; + }> { + const url = `${await this.discovery.getBaseUrl( + 'auth', + )}/.well-known/jwks.json`; + const response = await fetch(url); + + if (!response.ok) { + const payload = await response.text(); + const message = `Request failed with ${response.status} ${response.statusText}, ${payload}`; + throw new Error(message); + } + + const publicKeys: { keys: JSONWebKey[] } = await response.json(); + + return publicKeys; + } + + /** + * Fetches public keys and caches them locally + */ + private async refreshKeyStore(): Promise { + const now = Date.now() / 1000; + const publicKeys = await this.listPublicKeys(); + this.keyStore = JWKS.asKeyStore({ + keys: publicKeys.keys.map(key => key as JSONWebKey), + }); + this.keyStoreUpdated = now; + } +} diff --git a/plugins/auth-backend/src/identity/MemoryKeyStore.ts b/plugins/auth-backend/src/identity/MemoryKeyStore.ts new file mode 100644 index 0000000000..b35002b146 --- /dev/null +++ b/plugins/auth-backend/src/identity/MemoryKeyStore.ts @@ -0,0 +1,47 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { utc } from 'moment'; +import { KeyStore, AnyJWK, StoredKey } from './types'; + +export class MemoryKeyStore implements KeyStore { + private readonly keys = new Map< + string, + { createdAt: moment.Moment; key: string } + >(); + + async addKey(key: AnyJWK): Promise { + this.keys.set(key.kid, { + createdAt: utc(), + key: JSON.stringify(key), + }); + } + + async removeKeys(kids: string[]): Promise { + for (const kid of kids) { + this.keys.delete(kid); + } + } + + async listKeys(): Promise<{ items: StoredKey[] }> { + return { + items: Array.from(this.keys).map(([, { createdAt, key: keyStr }]) => ({ + createdAt, + key: JSON.parse(keyStr), + })), + }; + } +} diff --git a/plugins/auth-backend/src/identity/TokenFactory.test.ts b/plugins/auth-backend/src/identity/TokenFactory.test.ts index 8b719799b1..0c7febe616 100644 --- a/plugins/auth-backend/src/identity/TokenFactory.test.ts +++ b/plugins/auth-backend/src/identity/TokenFactory.test.ts @@ -14,43 +14,13 @@ * limitations under the License. */ -import { utc } from 'moment'; +import { MemoryKeyStore } from './MemoryKeyStore'; import { TokenFactory } from './TokenFactory'; import { getVoidLogger } from '@backstage/backend-common'; -import { KeyStore, AnyJWK, StoredKey } from './types'; import { JWKS, JSONWebKey, JWT } from 'jose'; const logger = getVoidLogger(); -class MemoryKeyStore implements KeyStore { - private readonly keys = new Map< - string, - { createdAt: moment.Moment; key: string } - >(); - - async addKey(key: AnyJWK): Promise { - this.keys.set(key.kid, { - createdAt: utc(), - key: JSON.stringify(key), - }); - } - - async removeKeys(kids: string[]): Promise { - for (const kid of kids) { - this.keys.delete(kid); - } - } - - async listKeys(): Promise<{ items: StoredKey[] }> { - return { - items: Array.from(this.keys).map(([, { createdAt, key: keyStr }]) => ({ - createdAt, - key: JSON.parse(keyStr), - })), - }; - } -} - function jwtKid(jwt: string): string { const { header } = JWT.decode(jwt, { complete: true }) as { header: { kid: string }; diff --git a/plugins/auth-backend/src/identity/index.ts b/plugins/auth-backend/src/identity/index.ts index 19eec94556..a76dcea440 100644 --- a/plugins/auth-backend/src/identity/index.ts +++ b/plugins/auth-backend/src/identity/index.ts @@ -15,6 +15,7 @@ */ export { createOidcRouter } from './router'; +export { IdentityClient } from './IdentityClient'; export { TokenFactory } from './TokenFactory'; export { DatabaseKeyStore } from './DatabaseKeyStore'; export type { KeyStore, TokenIssuer, TokenParams } from './types'; diff --git a/plugins/auth-backend/src/index.ts b/plugins/auth-backend/src/index.ts index 3d3f059dcc..a352362089 100644 --- a/plugins/auth-backend/src/index.ts +++ b/plugins/auth-backend/src/index.ts @@ -15,6 +15,7 @@ */ export * from './service/router'; +export { IdentityClient } from './identity'; export * from './providers'; // flow package provides 2 functions diff --git a/plugins/auth-backend/src/providers/aws-alb/index.ts b/plugins/auth-backend/src/providers/aws-alb/index.ts new file mode 100644 index 0000000000..f8b5c9e5d7 --- /dev/null +++ b/plugins/auth-backend/src/providers/aws-alb/index.ts @@ -0,0 +1,16 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +export { createAwsAlbProvider } from './provider'; diff --git a/plugins/auth-backend/src/providers/aws-alb/provider.test.ts b/plugins/auth-backend/src/providers/aws-alb/provider.test.ts new file mode 100644 index 0000000000..bae809971e --- /dev/null +++ b/plugins/auth-backend/src/providers/aws-alb/provider.test.ts @@ -0,0 +1,179 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { getVoidLogger } from '@backstage/backend-common'; +import express from 'express'; +import { JWT } from 'jose'; + +import { AwsAlbAuthProvider } from './provider'; +import { AuthResponse } from '../types'; + +const jwtMock = JWT as jest.Mocked; + +const mockKey = async () => { + return `-----BEGIN PUBLIC KEY----- +MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEnuN4LlaJhaUpx+qZFTzYCrSBLk0I +yOlxJ2VW88mLAQGJ7HPAvOdylxZsItMnzCuqNzZvie8m/NJsOjhDncVkrw== +-----END PUBLIC KEY----- +`; +}; + +jest.mock('jose'); + +jest.mock('cross-fetch', () => ({ + __esModule: true, + default: async () => { + return { + text: async () => { + return mockKey(); + }, + }; + }, +})); + +const identityResolutionCallbackMock = async (): Promise> => { + return { + backstageIdentity: { + id: 'foo', + idToken: '', + }, + profile: { + displayName: 'Foo Bar', + }, + providerInfo: {}, + }; +}; + +const identityResolutionCallbackRejectedMock = async (): Promise< + AuthResponse +> => { + throw new Error('failed'); +}; + +beforeEach(() => { + jest.clearAllMocks(); +}); + +describe('AwsALBAuthProvider', () => { + const catalogApi = { + /* eslint-disable-next-line @typescript-eslint/no-unused-vars */ + addLocation: jest.fn(), + getEntities: jest.fn(), + getLocationByEntity: jest.fn(), + getLocationById: jest.fn(), + removeEntityByUid: jest.fn(), + getEntityByName: jest.fn(), + }; + + const mockResponseSend = jest.fn(); + const mockRequest = ({ + header: jest.fn(() => { + return 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCIsImtpZCI6ImZvbyIsImlzc3VlciI6ImZvbyJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ.zUkMYAuMwC1T0tyHMpxXrkbFDa4aGhB8d9um_tI2hsI'; + }), + } as unknown) as express.Request; + const mockRequestWithoutJwt = ({ + header: jest.fn(() => { + return undefined; + }), + } as unknown) as express.Request; + const mockResponse = ({ + header: () => jest.fn(), + send: mockResponseSend, + } as unknown) as express.Response; + + describe('should transform to type OAuthResponse', () => { + it('when JWT is valid and identity is resolved successfully', async () => { + const provider = new AwsAlbAuthProvider(getVoidLogger(), catalogApi, { + region: 'us-west-2', + identityResolutionCallback: identityResolutionCallbackMock, + issuer: 'foo', + }); + + jwtMock.verify.mockImplementationOnce(() => ({ + sub: 'foo', + })); + + await provider.refresh(mockRequest, mockResponse); + + expect(mockResponseSend.mock.calls[0][0]).toEqual({ + backstageIdentity: { + id: 'foo', + idToken: '', + }, + profile: { + displayName: 'Foo Bar', + }, + providerInfo: {}, + }); + }); + }); + describe('should fail when', () => { + it('JWT is missing', async () => { + const provider = new AwsAlbAuthProvider(getVoidLogger(), catalogApi, { + region: 'us-west-2', + identityResolutionCallback: identityResolutionCallbackMock, + issuer: 'foo', + }); + + await provider.refresh(mockRequestWithoutJwt, mockResponse); + + expect(mockResponseSend.mock.calls[0][0]).toEqual(401); + }); + + it('JWT is invalid', async () => { + const provider = new AwsAlbAuthProvider(getVoidLogger(), catalogApi, { + region: 'us-west-2', + identityResolutionCallback: identityResolutionCallbackMock, + issuer: 'foo', + }); + + jwtMock.verify.mockImplementationOnce(() => { + throw new Error('bad JWT'); + }); + + await provider.refresh(mockRequest, mockResponse); + + expect(mockResponseSend.mock.calls[0][0]).toEqual(401); + }); + + it('issuer is invalid', async () => { + const provider = new AwsAlbAuthProvider(getVoidLogger(), catalogApi, { + region: 'us-west-2', + identityResolutionCallback: identityResolutionCallbackMock, + issuer: 'foobar', + }); + + jwtMock.verify.mockReturnValueOnce({}); + + await provider.refresh(mockRequest, mockResponse); + + expect(mockResponseSend.mock.calls[0][0]).toEqual(401); + }); + + it('identity resolution callback rejects', async () => { + const provider = new AwsAlbAuthProvider(getVoidLogger(), catalogApi, { + region: 'us-west-2', + identityResolutionCallback: identityResolutionCallbackRejectedMock, + issuer: 'foo', + }); + + jwtMock.verify.mockReturnValueOnce({}); + + await provider.refresh(mockRequest, mockResponse); + + expect(mockResponseSend.mock.calls[0][0]).toEqual(401); + }); + }); +}); diff --git a/plugins/auth-backend/src/providers/aws-alb/provider.ts b/plugins/auth-backend/src/providers/aws-alb/provider.ts new file mode 100644 index 0000000000..41dc5cf916 --- /dev/null +++ b/plugins/auth-backend/src/providers/aws-alb/provider.ts @@ -0,0 +1,130 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { + AuthProviderFactoryOptions, + AuthProviderRouteHandlers, + ExperimentalIdentityResolver, +} from '../types'; +import express from 'express'; +import fetch from 'cross-fetch'; +import * as crypto from 'crypto'; +import { KeyObject } from 'crypto'; +import { Logger } from 'winston'; +import NodeCache from 'node-cache'; +import { JWT } from 'jose'; +import { CatalogApi } from '@backstage/catalog-client'; + +const ALB_JWT_HEADER = 'x-amzn-oidc-data'; +/** + * A callback function that receives a verified JWT and returns a UserEntity + * @param {payload} The verified JWT payload + */ +type AwsAlbAuthProviderOptions = { + region: string; + issuer: string; + identityResolutionCallback: ExperimentalIdentityResolver; +}; +export const getJWTHeaders = (input: string) => { + const encoded = input.split('.')[0]; + return JSON.parse(Buffer.from(encoded, 'base64').toString('utf8')); +}; + +export class AwsAlbAuthProvider implements AuthProviderRouteHandlers { + private logger: Logger; + private readonly catalogClient: CatalogApi; + private options: AwsAlbAuthProviderOptions; + private readonly keyCache: NodeCache; + + constructor( + logger: Logger, + catalogClient: CatalogApi, + options: AwsAlbAuthProviderOptions, + ) { + this.logger = logger; + this.catalogClient = catalogClient; + this.options = options; + this.keyCache = new NodeCache({ stdTTL: 3600 }); + } + frameHandler(): Promise { + return Promise.resolve(undefined); + } + + async refresh(req: express.Request, res: express.Response): Promise { + const jwt = req.header(ALB_JWT_HEADER); + if (jwt !== undefined) { + try { + const headers = getJWTHeaders(jwt); + const key = await this.getKey(headers.kid); + const payload = JWT.verify(jwt, key); + + if ( + this.options.issuer !== '' && + headers.issuer !== this.options.issuer + ) { + throw new Error('issuer mismatch on JWT'); + } + + const resolvedEntity = await this.options.identityResolutionCallback( + payload, + this.catalogClient, + ); + res.send(resolvedEntity); + } catch (e) { + this.logger.error('exception occurred during JWT processing', e); + res.send(401); + } + } else { + res.send(401); + } + } + + start(): Promise { + return Promise.resolve(undefined); + } + + async getKey(keyId: string): Promise { + const optionalCacheKey = this.keyCache.get(keyId); + if (optionalCacheKey) { + return optionalCacheKey; + } + const keyText: string = await fetch( + `https://public-keys.auth.elb.${this.options.region}.amazonaws.com/${keyId}`, + ).then(response => response.text()); + const keyValue = crypto.createPublicKey(keyText); + this.keyCache.set(keyId, keyValue); + return keyValue; + } +} + +export const createAwsAlbProvider = ({ + logger, + catalogApi, + config, + identityResolver, +}: AuthProviderFactoryOptions) => { + const region = config.getString('region'); + const issuer = config.getString('iss'); + if (identityResolver !== undefined) { + return new AwsAlbAuthProvider(logger, catalogApi, { + region, + issuer, + identityResolutionCallback: identityResolver, + }); + } + throw new Error( + 'Identity resolver is required to use this authentication provider', + ); +}; diff --git a/plugins/auth-backend/src/providers/factories.ts b/plugins/auth-backend/src/providers/factories.ts index 7bad4f4d81..619fb1c706 100644 --- a/plugins/auth-backend/src/providers/factories.ts +++ b/plugins/auth-backend/src/providers/factories.ts @@ -25,6 +25,7 @@ import { createAuth0Provider } from './auth0'; import { createMicrosoftProvider } from './microsoft'; import { createOneLoginProvider } from './onelogin'; import { AuthProviderFactory } from './types'; +import { createAwsAlbProvider } from './aws-alb'; export const factories: { [providerId: string]: AuthProviderFactory } = { google: createGoogleProvider, @@ -37,4 +38,5 @@ export const factories: { [providerId: string]: AuthProviderFactory } = { oauth2: createOAuth2Provider, oidc: createOidcProvider, onelogin: createOneLoginProvider, + awsalb: createAwsAlbProvider, }; diff --git a/plugins/auth-backend/src/providers/oidc/provider.test.ts b/plugins/auth-backend/src/providers/oidc/provider.test.ts index f2bd8dcb90..fcbe9d7abc 100644 --- a/plugins/auth-backend/src/providers/oidc/provider.test.ts +++ b/plugins/auth-backend/src/providers/oidc/provider.test.ts @@ -71,6 +71,7 @@ describe('OidcAuthProvider', () => { const jwt = { sub: 'alice', iss: 'https://oidc.test', + iat: Date.now(), aud: clientMetadata.clientId, exp: Date.now() + 10000, }; diff --git a/plugins/auth-backend/src/providers/types.ts b/plugins/auth-backend/src/providers/types.ts index a40f2a96ee..1a4e7b118a 100644 --- a/plugins/auth-backend/src/providers/types.ts +++ b/plugins/auth-backend/src/providers/types.ts @@ -112,6 +112,19 @@ export interface AuthProviderRouteHandlers { logout?(req: express.Request, res: express.Response): Promise; } +/** + * EXPERIMENTAL - this will almost certainly break in a future release. + * + * Used to resolve an identity from auth information in some auth providers. + */ +export type ExperimentalIdentityResolver = ( + /** + * An object containing information specific to the auth provider. + */ + payload: object, + catalogApi: CatalogApi, +) => Promise>; + export type AuthProviderFactoryOptions = { providerId: string; globalConfig: AuthProviderConfig; @@ -120,6 +133,7 @@ export type AuthProviderFactoryOptions = { tokenIssuer: TokenIssuer; discovery: PluginEndpointDiscovery; catalogApi: CatalogApi; + identityResolver?: ExperimentalIdentityResolver; }; export type AuthProviderFactory = ( diff --git a/plugins/catalog-backend/CHANGELOG.md b/plugins/catalog-backend/CHANGELOG.md index a3ee4b7eb3..77620c1cea 100644 --- a/plugins/catalog-backend/CHANGELOG.md +++ b/plugins/catalog-backend/CHANGELOG.md @@ -1,5 +1,48 @@ # @backstage/plugin-catalog-backend +## 0.5.4 + +### Patch Changes + +- def2307f3: Adds a `backstage.io/managed-by-origin-location` annotation to all entities. It links to the + location that was registered to the catalog and which emitted this entity. It has a different + semantic than the existing `backstage.io/managed-by-location` annotation, which tells the direct + parent location that created this entity. + + Consider this example: The Backstage operator adds a location of type `github-org` in the + `app-config.yaml`. This setting will be added to a `bootstrap:boostrap` location. The processor + discovers the entities in the following branch + `Location bootstrap:bootstrap -> Location github-org:… -> User xyz`. The user `xyz` will be: + + ```yaml + apiVersion: backstage.io/v1alpha1 + kind: User + metadata: + name: xyz + annotations: + # This entity was added by the 'github-org:…' location + backstage.io/managed-by-location: github-org:… + # The entity was added because the 'bootstrap:boostrap' was added to the catalog + backstage.io/managed-by-origin-location: bootstrap:bootstrap + # ... + spec: + # ... + ``` + +- 318a6af9f: Change AWS Account type from Component to Resource +- ac7be581a: Refuse to remove the bootstrap location +- ad838c02f: Reduce log noise on locations refresh +- f9ba00a1c: Update the @azure/msal-node dependency to 1.0.0-beta.3. +- Updated dependencies [def2307f3] +- Updated dependencies [0b135e7e0] +- Updated dependencies [294a70cab] +- Updated dependencies [0ea032763] +- Updated dependencies [5345a1f98] +- Updated dependencies [09a370426] +- Updated dependencies [a93f42213] + - @backstage/catalog-model@0.7.0 + - @backstage/backend-common@0.5.0 + ## 0.5.3 ### Patch Changes diff --git a/plugins/catalog-backend/package.json b/plugins/catalog-backend/package.json index e6b6713e20..b2db27770b 100644 --- a/plugins/catalog-backend/package.json +++ b/plugins/catalog-backend/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-catalog-backend", - "version": "0.5.3", + "version": "0.5.4", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -31,8 +31,8 @@ "dependencies": { "@aws-sdk/client-organizations": "^3.2.0", "@azure/msal-node": "^1.0.0-beta.3", - "@backstage/backend-common": "^0.4.3", - "@backstage/catalog-model": "^0.6.1", + "@backstage/backend-common": "^0.5.0", + "@backstage/catalog-model": "^0.7.0", "@backstage/config": "^0.1.2", "@octokit/graphql": "^4.5.8", "@types/express": "^4.17.6", @@ -57,7 +57,7 @@ "yup": "^0.29.3" }, "devDependencies": { - "@backstage/cli": "^0.4.6", + "@backstage/cli": "^0.4.7", "@backstage/test-utils": "^0.1.6", "@types/core-js": "^2.5.4", "@types/git-url-parse": "^9.0.0", diff --git a/plugins/catalog-backend/src/ingestion/LocationReaders.ts b/plugins/catalog-backend/src/ingestion/LocationReaders.ts index a9248bb3fb..cb703b8e15 100644 --- a/plugins/catalog-backend/src/ingestion/LocationReaders.ts +++ b/plugins/catalog-backend/src/ingestion/LocationReaders.ts @@ -78,13 +78,17 @@ export class LocationReaders implements LocationReader { if (rulesEnforcer.isAllowed(item.entity, item.location)) { const relations = Array(); - const entity = await this.handleEntity(item, emitResult => { - if (emitResult.type === 'relation') { - relations.push(emitResult.relation); - return; - } - emit(emitResult); - }); + const entity = await this.handleEntity( + item, + emitResult => { + if (emitResult.type === 'relation') { + relations.push(emitResult.relation); + return; + } + emit(emitResult); + }, + location, + ); if (entity) { output.entities.push({ @@ -165,6 +169,7 @@ export class LocationReaders implements LocationReader { private async handleEntity( item: CatalogProcessorEntityResult, emit: CatalogProcessorEmit, + originLocation: LocationSpec, ): Promise { const { processors, logger } = this.options; @@ -185,6 +190,7 @@ export class LocationReaders implements LocationReader { current, item.location, emit, + originLocation, ); } catch (e) { const message = `Processor ${processor.constructor.name} threw an error while preprocessing entity ${kind}:${namespace}/${name} at ${item.location.type} ${item.location.target}, ${e}`; diff --git a/plugins/catalog-backend/src/ingestion/processors/AnnotateLocationEntityProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/AnnotateLocationEntityProcessor.test.ts new file mode 100644 index 0000000000..cfd98e9e6a --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/AnnotateLocationEntityProcessor.test.ts @@ -0,0 +1,62 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Entity, LocationSpec } from '@backstage/catalog-model'; +import { AnnotateLocationEntityProcessor } from './AnnotateLocationEntityProcessor'; + +describe('AnnotateLocationEntityProcessor', () => { + describe('preProcessEntity', () => { + it('adds annotations', async () => { + const entity: Entity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Component', + metadata: { + name: 'my-component', + }, + }; + + const location: LocationSpec = { + type: 'url', + target: 'my-location', + }; + const originLocation: LocationSpec = { + type: 'url', + target: 'my-origin-location', + }; + + const processor = new AnnotateLocationEntityProcessor(); + + expect( + await processor.preProcessEntity( + entity, + location, + () => {}, + originLocation, + ), + ).toEqual({ + apiVersion: 'backstage.io/v1alpha1', + kind: 'Component', + metadata: { + name: 'my-component', + annotations: { + 'backstage.io/managed-by-location': 'url:my-location', + 'backstage.io/managed-by-origin-location': 'url:my-origin-location', + }, + }, + }); + }); + }); +}); diff --git a/plugins/catalog-backend/src/ingestion/processors/AnnotateLocationEntityProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/AnnotateLocationEntityProcessor.ts index ea8afcddc5..b40378226a 100644 --- a/plugins/catalog-backend/src/ingestion/processors/AnnotateLocationEntityProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/AnnotateLocationEntityProcessor.ts @@ -14,20 +14,28 @@ * limitations under the License. */ -import { Entity, LocationSpec } from '@backstage/catalog-model'; +import { + Entity, + LOCATION_ANNOTATION, + LocationSpec, + ORIGIN_LOCATION_ANNOTATION, +} from '@backstage/catalog-model'; import lodash from 'lodash'; -import { CatalogProcessor } from './types'; +import { CatalogProcessor, CatalogProcessorEmit } from './types'; export class AnnotateLocationEntityProcessor implements CatalogProcessor { async preProcessEntity( entity: Entity, location: LocationSpec, + _: CatalogProcessorEmit, + originLocation: LocationSpec, ): Promise { return lodash.merge( { metadata: { annotations: { - 'backstage.io/managed-by-location': `${location.type}:${location.target}`, + [LOCATION_ANNOTATION]: `${location.type}:${location.target}`, + [ORIGIN_LOCATION_ANNOTATION]: `${originLocation.type}:${originLocation.target}`, }, }, }, diff --git a/plugins/catalog-backend/src/ingestion/processors/AwsOrganizationCloudAccountProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/AwsOrganizationCloudAccountProcessor.test.ts index 5044a16aea..6eb7a45c63 100644 --- a/plugins/catalog-backend/src/ingestion/processors/AwsOrganizationCloudAccountProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/AwsOrganizationCloudAccountProcessor.test.ts @@ -45,7 +45,7 @@ describe('AwsOrganizationCloudAccountProcessor', () => { location, entity: { apiVersion: 'backstage.io/v1alpha1', - kind: 'Component', + kind: 'Resource', metadata: { annotations: { 'amazonaws.com/arn': @@ -58,7 +58,6 @@ describe('AwsOrganizationCloudAccountProcessor', () => { }, spec: { type: 'cloud-account', - lifecycle: 'unknown', owner: 'unknown', }, }, @@ -91,7 +90,7 @@ describe('AwsOrganizationCloudAccountProcessor', () => { location, entity: { apiVersion: 'backstage.io/v1alpha1', - kind: 'Component', + kind: 'Resource', metadata: { annotations: { 'amazonaws.com/arn': @@ -104,7 +103,6 @@ describe('AwsOrganizationCloudAccountProcessor', () => { }, spec: { type: 'cloud-account', - lifecycle: 'unknown', owner: 'unknown', }, }, diff --git a/plugins/catalog-backend/src/ingestion/processors/AwsOrganizationCloudAccountProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/AwsOrganizationCloudAccountProcessor.ts index 298fcb82ec..40516f88f7 100644 --- a/plugins/catalog-backend/src/ingestion/processors/AwsOrganizationCloudAccountProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/AwsOrganizationCloudAccountProcessor.ts @@ -13,10 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { - ComponentEntityV1alpha1, - LocationSpec, -} from '@backstage/catalog-model'; +import { LocationSpec, ResourceEntityV1alpha1 } from '@backstage/catalog-model'; import { Account, Organizations, @@ -82,13 +79,13 @@ export class AwsOrganizationCloudAccountProcessor implements CatalogProcessor { return awsAccounts; } - mapAccountToComponent(account: Account): ComponentEntityV1alpha1 { + mapAccountToComponent(account: Account): ResourceEntityV1alpha1 { const { accountId, organizationId } = this.extractInformationFromArn( account.Arn as string, ); return { apiVersion: 'backstage.io/v1alpha1', - kind: 'Component', + kind: 'Resource', metadata: { annotations: { [ACCOUNTID_ANNOTATION]: accountId, @@ -100,7 +97,6 @@ export class AwsOrganizationCloudAccountProcessor implements CatalogProcessor { }, spec: { type: 'cloud-account', - lifecycle: 'unknown', owner: 'unknown', }, }; @@ -129,7 +125,7 @@ export class AwsOrganizationCloudAccountProcessor implements CatalogProcessor { } return true; }) - .forEach((entity: ComponentEntityV1alpha1) => { + .forEach((entity: ResourceEntityV1alpha1) => { emit(results.entity(location, entity)); }); diff --git a/plugins/catalog-backend/src/ingestion/processors/types.ts b/plugins/catalog-backend/src/ingestion/processors/types.ts index 5da3510f7a..1bf30567bc 100644 --- a/plugins/catalog-backend/src/ingestion/processors/types.ts +++ b/plugins/catalog-backend/src/ingestion/processors/types.ts @@ -46,12 +46,16 @@ export type CatalogProcessor = { * @param entity The (possibly partial) entity to process * @param location The location that the entity came from * @param emit A sink for auxiliary items resulting from the processing + * @param originLocation The location that the entity originally came from. + * While location resolves to the direct parent location, originLocation + * tells which location was used to start the ingestion loop. * @returns The same entity or a modified version of it */ preProcessEntity?( entity: Entity, location: LocationSpec, emit: CatalogProcessorEmit, + originLocation: LocationSpec, ): Promise; /** diff --git a/plugins/catalog-graphql/CHANGELOG.md b/plugins/catalog-graphql/CHANGELOG.md index ceee335db0..ad536e45dc 100644 --- a/plugins/catalog-graphql/CHANGELOG.md +++ b/plugins/catalog-graphql/CHANGELOG.md @@ -1,5 +1,19 @@ # @backstage/plugin-catalog-graphql +## 0.2.6 + +### Patch Changes + +- Updated dependencies [def2307f3] +- Updated dependencies [0b135e7e0] +- Updated dependencies [294a70cab] +- Updated dependencies [0ea032763] +- Updated dependencies [5345a1f98] +- Updated dependencies [09a370426] +- Updated dependencies [a93f42213] + - @backstage/catalog-model@0.7.0 + - @backstage/backend-common@0.5.0 + ## 0.2.5 ### Patch Changes diff --git a/plugins/catalog-graphql/package.json b/plugins/catalog-graphql/package.json index fb013f53c3..4533a541f0 100644 --- a/plugins/catalog-graphql/package.json +++ b/plugins/catalog-graphql/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-catalog-graphql", - "version": "0.2.5", + "version": "0.2.6", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -30,8 +30,8 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@backstage/backend-common": "^0.4.1", - "@backstage/catalog-model": "^0.6.0", + "@backstage/backend-common": "^0.5.0", + "@backstage/catalog-model": "^0.7.0", "@backstage/config": "^0.1.2", "@graphql-modules/core": "^0.7.17", "apollo-server": "^2.16.1", @@ -42,7 +42,7 @@ "winston": "^3.2.1" }, "devDependencies": { - "@backstage/cli": "^0.4.2", + "@backstage/cli": "^0.4.7", "@backstage/test-utils": "^0.1.5", "@graphql-codegen/cli": "^1.17.7", "@graphql-codegen/typescript": "^1.17.7", diff --git a/plugins/catalog-import/CHANGELOG.md b/plugins/catalog-import/CHANGELOG.md index a307095014..2d75ae6af4 100644 --- a/plugins/catalog-import/CHANGELOG.md +++ b/plugins/catalog-import/CHANGELOG.md @@ -1,5 +1,25 @@ # @backstage/plugin-catalog-import +## 0.3.5 + +### Patch Changes + +- 2b514d532: Modifying import functionality to register existing catalog-info.yaml if one exists in given GitHub repository +- Updated dependencies [def2307f3] +- Updated dependencies [efd6ef753] +- Updated dependencies [0b135e7e0] +- Updated dependencies [593632f07] +- Updated dependencies [33846acfc] +- Updated dependencies [fa8ba330a] +- Updated dependencies [a187b8ad0] +- Updated dependencies [ed6baab66] +- Updated dependencies [f04db53d7] +- Updated dependencies [a93f42213] + - @backstage/catalog-model@0.7.0 + - @backstage/core@0.5.0 + - @backstage/integration@0.3.0 + - @backstage/plugin-catalog@0.2.12 + ## 0.3.4 ### Patch Changes diff --git a/plugins/catalog-import/package.json b/plugins/catalog-import/package.json index 5e601ad896..b2c9991bb8 100644 --- a/plugins/catalog-import/package.json +++ b/plugins/catalog-import/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-catalog-import", - "version": "0.3.4", + "version": "0.3.5", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -30,10 +30,10 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@backstage/catalog-model": "^0.6.1", - "@backstage/core": "^0.4.4", - "@backstage/plugin-catalog": "^0.2.11", - "@backstage/integration": "^0.2.0", + "@backstage/catalog-model": "^0.7.0", + "@backstage/core": "^0.5.0", + "@backstage/integration": "^0.3.0", + "@backstage/plugin-catalog": "^0.2.12", "@backstage/theme": "^0.2.2", "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", @@ -49,8 +49,8 @@ "yaml": "^1.10.0" }, "devDependencies": { - "@backstage/cli": "^0.4.6", - "@backstage/dev-utils": "^0.1.7", + "@backstage/cli": "^0.4.7", + "@backstage/dev-utils": "^0.1.8", "@backstage/test-utils": "^0.1.6", "@testing-library/jest-dom": "^5.10.1", "@testing-library/react": "^10.4.1", diff --git a/plugins/catalog-import/src/api/CatalogImportApi.ts b/plugins/catalog-import/src/api/CatalogImportApi.ts index 99526a3eaa..5abb0e3e53 100644 --- a/plugins/catalog-import/src/api/CatalogImportApi.ts +++ b/plugins/catalog-import/src/api/CatalogImportApi.ts @@ -30,6 +30,11 @@ export interface CatalogImportApi { fileContent: string; githubIntegrationConfig: GitHubIntegrationConfig; }): Promise<{ link: string; location: string }>; + checkForExistingCatalogInfo(options: { + owner: string; + repo: string; + githubIntegrationConfig: GitHubIntegrationConfig; + }): Promise<{ exists: boolean; url?: string }>; createRepositoryLocation(options: { location: string }): Promise; generateEntityDefinitions(options: { repo: string; diff --git a/plugins/catalog-import/src/api/CatalogImportClient.test.ts b/plugins/catalog-import/src/api/CatalogImportClient.test.ts new file mode 100644 index 0000000000..0e0cf4b323 --- /dev/null +++ b/plugins/catalog-import/src/api/CatalogImportClient.test.ts @@ -0,0 +1,64 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { CatalogImportClient } from './CatalogImportClient'; + +jest.mock('@octokit/rest', () => ({ + Octokit: jest.fn().mockImplementation(() => { + return { + repos: { + get: () => + Promise.resolve({ + data: { + default_branch: 'main', + }, + }), + }, + search: { + code: () => + Promise.resolve({ + data: { + total_count: 2, + items: [ + { path: 'simple/path/catalog-info.yaml' }, + { path: 'co/mple/x/path/catalog-info.yaml' }, + { path: 'catalog-info.yaml' }, + ], + }, + }), + }, + }; + }), +})); + +describe('CatalogImportClient', () => { + describe('checkForExistingCatalogInfo', () => { + const cic = new CatalogImportClient({ + discoveryApi: { getBaseUrl: () => Promise.resolve('base') }, + githubAuthApi: { getAccessToken: (_, __) => Promise.resolve('token') }, + configApi: {} as any, + }); + it('should return the closest-to-root catalog-info from multiple responses', async () => { + const respo = await cic.checkForExistingCatalogInfo({ + owner: 'test-user', + repo: 'rest-repo', + githubIntegrationConfig: { host: 'https://github.com' }, + }); + expect(respo.exists).toBe(true); + expect(respo.url).toBe('blob/main/catalog-info.yaml'); + }); + }); +}); diff --git a/plugins/catalog-import/src/api/CatalogImportClient.ts b/plugins/catalog-import/src/api/CatalogImportClient.ts index a578f37a96..d1509c2105 100644 --- a/plugins/catalog-import/src/api/CatalogImportClient.ts +++ b/plugins/catalog-import/src/api/CatalogImportClient.ts @@ -90,6 +90,52 @@ export class CatalogImportClient implements CatalogImportApi { } } + async checkForExistingCatalogInfo({ + owner, + repo, + githubIntegrationConfig, + }: { + owner: string; + repo: string; + githubIntegrationConfig: GitHubIntegrationConfig; + }): Promise<{ exists: boolean; url?: string }> { + const token = await this.githubAuthApi.getAccessToken(['repo']); + const octo = new Octokit({ + auth: token, + baseUrl: githubIntegrationConfig.apiBaseUrl, + }); + const catalogFileName = 'catalog-info.yaml'; + const query = `repo:${owner}/${repo}+filename:${catalogFileName}`; + + const searchResult = await octo.search.code({ q: query }).catch(e => { + throw new Error( + formatHttpErrorMessage( + "Couldn't search repository for metadata file.", + e, + ), + ); + }); + const exists = searchResult.data.total_count > 0; + if (exists) { + const repoInformation = await octo.repos.get({ owner, repo }).catch(e => { + throw new Error(formatHttpErrorMessage("Couldn't fetch repo data", e)); + }); + const defaultBranch = repoInformation.data.default_branch; + + // Github search sorts returned values with 'best match' using 'multiple factors to boost the most relevant item', + // aka magic. + // Sorting to use the shortest item, closest to the repository root. + const catalogInfoItem = searchResult.data.items + .map(it => it.path) + .sort((a, b) => a.length - b.length)[0]; + return { + url: `blob/${defaultBranch}/${catalogInfoItem}`, + exists, + }; + } + return { exists }; + } + async submitPrToRepo({ owner, repo, diff --git a/plugins/catalog-import/src/components/ImportComponentForm.test.tsx b/plugins/catalog-import/src/components/ImportComponentForm.test.tsx new file mode 100644 index 0000000000..91a5208bee --- /dev/null +++ b/plugins/catalog-import/src/components/ImportComponentForm.test.tsx @@ -0,0 +1,89 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import React from 'react'; +import { renderInTestApp } from '@backstage/test-utils'; +import { RegisterComponentForm } from './ImportComponentForm'; +import { + ApiProvider, + ApiRegistry, + DiscoveryApi, + errorApiRef, +} from '@backstage/core'; +import { catalogApiRef, CatalogClient } from '@backstage/plugin-catalog'; +import { catalogImportApiRef, CatalogImportClient } from '../api'; +import { fireEvent, waitFor, screen } from '@testing-library/react'; + +describe('', () => { + let apis: ApiRegistry; + + const mockErrorApi: jest.Mocked = { + post: jest.fn(), + error$: jest.fn(), + }; + + beforeEach(() => { + apis = ApiRegistry.from([ + [catalogApiRef, new CatalogClient({ discoveryApi: {} as DiscoveryApi })], + [ + catalogImportApiRef, + new CatalogImportClient({ + discoveryApi: { getBaseUrl: () => Promise.resolve('base') }, + githubAuthApi: { + getAccessToken: (_, __) => Promise.resolve('token'), + }, + configApi: {} as any, + }), + ], + [errorApiRef, mockErrorApi], + ]); + }); + + async function renderSUT( + nextStep: () => void = () => {}, + saveConfig: () => void = () => {}, + ) { + return await renderInTestApp( + + + , + ); + } + + it('Renders without exploding', async () => { + await renderSUT(); + expect( + screen.getByPlaceholderText('https://github.com/backstage/backstage'), + ).toBeInTheDocument(); + }); + + it('Should have basic URL validation for input', async () => { + await renderSUT(); + await waitFor(() => { + fireEvent.input( + screen.getByPlaceholderText('https://github.com/backstage/backstage'), + { target: { value: 'not a url' } }, + ); + }); + await waitFor(() => { + fireEvent.click(screen.getByText('Next')); + }); + expect(screen.getByText('Must start with https://.')).toBeInTheDocument(); + }); +}); diff --git a/plugins/catalog-import/src/components/ImportComponentForm.tsx b/plugins/catalog-import/src/components/ImportComponentForm.tsx index 7434f07b10..c233be5a8c 100644 --- a/plugins/catalog-import/src/components/ImportComponentForm.tsx +++ b/plugins/catalog-import/src/components/ImportComponentForm.tsx @@ -64,27 +64,45 @@ export const RegisterComponentForm = ({ const isMounted = useMountedState(); const errorApi = useApi(errorApiRef); - const { generateEntityDefinitions } = useGithubRepos(); + const { + generateEntityDefinitions, + checkForExistingCatalogInfo, + } = useGithubRepos(); const onSubmit = async (formData: Record) => { const { componentLocation: target } = formData; - try { - if (!isMounted()) return; - const type = urlType(target); + async function saveCatalogFileConfig(target: string) { + const data = await catalogApi.addLocation({ target }); + saveConfig({ + type: 'file', + location: data.location.target, + config: data.entities, + }); + } - if (type === 'tree') { + async function trySaveRepositoryConfig(target: string) { + const existingCatalog = await checkForExistingCatalogInfo(target); + if (existingCatalog.exists) { + const targetUrl = target.endsWith('/') + ? `${target}${existingCatalog.url}` + : `${target}/${existingCatalog.url}`; + await saveCatalogFileConfig(targetUrl); + } else { saveConfig({ - type, + type: 'tree', location: target, config: await generateEntityDefinitions(target), }); + } + } + + try { + if (!isMounted()) return; + const type = urlType(target); + if (type === 'tree') { + await trySaveRepositoryConfig(target); } else { - const data = await catalogApi.addLocation({ target }); - saveConfig({ - type, - location: data.location.target, - config: data.entities, - }); + await saveCatalogFileConfig(target); } nextStep(); } catch (e) { diff --git a/plugins/catalog-import/src/components/ImportComponentPage.test.tsx b/plugins/catalog-import/src/components/ImportComponentPage.test.tsx new file mode 100644 index 0000000000..932f9aafa0 --- /dev/null +++ b/plugins/catalog-import/src/components/ImportComponentPage.test.tsx @@ -0,0 +1,267 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import React from 'react'; +import { msw, renderInTestApp } from '@backstage/test-utils'; +import { ImportComponentPage } from './ImportComponentPage'; +import { + ApiProvider, + ApiRegistry, + configApiRef, + errorApiRef, +} from '@backstage/core'; +import { catalogApiRef, CatalogClient } from '@backstage/plugin-catalog'; +import { catalogImportApiRef, CatalogImportClient } from '../api'; + +import { fireEvent, screen, waitFor } from '@testing-library/react'; + +import { rest } from 'msw'; +import { setupServer } from 'msw/node'; + +let codeSearchMockResponse: () => Promise<{ + data: { + total_count: number; + items: Array<{ path: string }>; + }; +}>; + +let findGithubConfigMockResponse = () => ({ + host: 'test.localhost', + owner: 'someuser', +}); + +jest.mock('@backstage/integration', () => ({ + readGitHubIntegrationConfigs: () => ({ + find: findGithubConfigMockResponse, + }), +})); + +jest.mock('@octokit/rest', () => ({ + Octokit: jest.fn().mockImplementation(() => { + return { + repos: { + get: () => + Promise.resolve({ + data: { + default_branch: 'main', + }, + }), + }, + search: { + code: codeSearchMockResponse, + }, + }; + }), +})); + +const OUR_GITHUB_TEST_REPO = 'https://github.com/someuser/somerepo'; +describe('', () => { + const server = setupServer(); + msw.setupDefaultHandlers(server); + + beforeEach(() => { + server.use( + rest.post('https://backend.localhost/locations', (_, res, ctx) => { + return res( + ctx.status(201), + ctx.json(require('../mocks/locations-POST-response.json')), + ); + }), + rest.post('https://backend.localhost/analyze-location', (_, res, ctx) => { + return res( + ctx.json(require('../mocks/analyze-location-POST-response.json')), + ); + }), + ); + }); + beforeAll(() => server.listen()); + afterEach(() => server.resetHandlers()); + afterAll(() => server.close()); + + let apis: ApiRegistry; + + const mockErrorApi: jest.Mocked = { + post: jest.fn(), + error$: jest.fn(), + }; + + beforeEach(() => { + const getBaseUrl = () => Promise.resolve('https://backend.localhost'); + apis = ApiRegistry.from([ + [ + catalogApiRef, + new CatalogClient({ + discoveryApi: { getBaseUrl }, + }), + ], + [ + catalogImportApiRef, + new CatalogImportClient({ + discoveryApi: { getBaseUrl }, + githubAuthApi: { + getAccessToken: (_, __) => Promise.resolve('token'), + }, + configApi: {} as any, + }), + ], + [ + configApiRef, + { + getOptional: () => 'Title', + getOptionalConfigArray: () => [], + has: () => true, + getConfig: () => ({ + has: () => true, + }), + }, + ], + [errorApiRef, mockErrorApi], + ]); + }); + + async function renderSUT() { + return await renderInTestApp( + + + , + ); + } + + it('Should not explode on non-Github URLs', async () => { + findGithubConfigMockResponse = () => undefined!!; + await renderSUT(); + await waitFor(() => { + fireEvent.input( + screen.getByPlaceholderText('https://github.com/backstage/backstage'), + { + target: { + value: 'https://test-git-provider.localhost/someuser/somerepo', + }, + }, + ); + }); + + fireEvent.click(screen.getByText('Next')); + await waitFor(() => { + const firstStepInput = screen.queryByPlaceholderText( + 'https://github.com/backstage/backstage', + ); + expect(firstStepInput).not.toBeInTheDocument(); + }); + }); + + it('Should offer direct file import from non-Github URLs', async () => { + findGithubConfigMockResponse = () => undefined!!; + await renderSUT(); + await waitFor(() => { + fireEvent.input( + screen.getByPlaceholderText('https://github.com/backstage/backstage'), + { + target: { + value: + 'https://test-git-provider.localhost/someuser/somerepo/catalog-info.yaml', + }, + }, + ); + }); + + fireEvent.click(screen.getByText('Next')); + await waitFor(() => { + const firstStepInput = screen.queryByPlaceholderText( + 'https://github.com/backstage/backstage', + ); + expect(firstStepInput).not.toBeInTheDocument(); + }); + expect( + screen.getByText( + 'https://test-git-provider.localhost/someuser/somerepo/catalog-info.yaml', + ), + ).toBeInTheDocument(); + }); + + it('Should use found yaml file directly and not create a pull request if GitHub api returns one', async () => { + findGithubConfigMockResponse = () => ({ + host: 'test.localhost', + owner: 'someuser', + }); + codeSearchMockResponse = () => + Promise.resolve({ + data: { + total_count: 3, + items: [ + { path: 'simple/path/catalog-info.yaml' }, + { path: 'co/mple/x/path/catalog-info.yaml' }, + { path: 'catalog-info.yaml' }, + ], + }, + }); + await renderSUT(); + await waitFor(() => { + fireEvent.input( + screen.getByPlaceholderText('https://github.com/backstage/backstage'), + { target: { value: OUR_GITHUB_TEST_REPO } }, + ); + }); + + fireEvent.click(screen.getByText('Next')); + await waitFor(() => { + expect( + screen.getByText( + 'https://github.com/someusername/somerepo/blob/master/src/catalog-info.yaml', + ), + ).toBeInTheDocument(); + + const pullReqText = screen.queryByText('pull request'); + expect(pullReqText).not.toBeInTheDocument(); + }); + }); + + it('Should indicate a pull request creation when no yaml file found in the repo', async () => { + findGithubConfigMockResponse = () => ({ + host: 'test.localhost', + owner: 'someuser', + }); + codeSearchMockResponse = () => + Promise.resolve({ + data: { + total_count: 0, + items: [], + }, + }); + const { container } = await renderSUT(); + await waitFor(() => { + fireEvent.input( + screen.getByPlaceholderText('https://github.com/backstage/backstage'), + { target: { value: OUR_GITHUB_TEST_REPO } }, + ); + }); + + fireEvent.click(screen.getByText('Next')); + await waitFor(() => { + expect(screen.getByText(OUR_GITHUB_TEST_REPO)).toBeInTheDocument(); + }); + const textNode = container + .querySelector('a[href="https://github.com/someuser/somerepo"]') + ?.closest('p'); + expect(textNode?.innerHTML).toContain( + 'Following config object will be submitted in a pull request to the repository', + ); + expect( + screen.queryByText( + 'https://github.com/someusername/somerepo/blob/master/src/catalog-info.yaml', + ), + ).not.toBeInTheDocument(); + }); +}); diff --git a/plugins/catalog-import/src/components/ImportComponentPage.tsx b/plugins/catalog-import/src/components/ImportComponentPage.tsx index 7e0d0ff92e..e284e95456 100644 --- a/plugins/catalog-import/src/components/ImportComponentPage.tsx +++ b/plugins/catalog-import/src/components/ImportComponentPage.tsx @@ -109,10 +109,11 @@ export const ImportComponentPage = ({ GitHub Repo - If you already have code in a GitHub repository, enter the - full URL to your repo and a new pull request with a sample - Backstage metadata Entity File ( - catalog-info.yaml) will be opened for you. + If you already have code in a GitHub repository without + Backstage metadata file set up for it, enter the full URL to + your repo and a new pull request with a sample Backstage + metadata Entity File (catalog-info.yaml) will + be opened for you. )} diff --git a/plugins/catalog-import/src/mocks/analyze-location-POST-response.json b/plugins/catalog-import/src/mocks/analyze-location-POST-response.json new file mode 100644 index 0000000000..eaadf1054a --- /dev/null +++ b/plugins/catalog-import/src/mocks/analyze-location-POST-response.json @@ -0,0 +1,22 @@ +{ + "existingEntityFiles": [], + "generateEntities": [ + { + "entity": { + "apiVersion": "backstage.io/v1alpha1", + "kind": "Component", + "metadata": { + "name": "somerepo", + "annotations": { + "github.com/project-slug": "someuser/somerepo" + } + }, + "spec": { + "type": "other", + "lifecycle": "unknown" + } + }, + "fields": [] + } + ] +} diff --git a/plugins/catalog-import/src/mocks/locations-POST-response.json b/plugins/catalog-import/src/mocks/locations-POST-response.json new file mode 100644 index 0000000000..c44a0f4d61 --- /dev/null +++ b/plugins/catalog-import/src/mocks/locations-POST-response.json @@ -0,0 +1,39 @@ +{ + "location": { + "id": "d4a64359-a709-4c91-a9de-0905a033bf22", + "type": "url", + "target": "https://github.com/someusername/somerepo/blob/master/src/catalog-info.yaml" + }, + "entities": [ + { + "metadata": { + "namespace": "default", + "annotations": { + "backstage.io/managed-by-location": "url:https://github.com/someusername/somerepo/blob/master/src/catalog-info.yaml", + "github.com/project-slug": "someusername/somerepo" + }, + "name": "somerepo", + "uid": "e992d5ee-7c70-4316-90cf-325f1a0a5146", + "etag": "YWE2M2Q5MzgtNjdkNi00N2QwLWJkZjYtNDM0MTMzMDI4Y2I0", + "generation": 1 + }, + "apiVersion": "backstage.io/v1alpha1", + "kind": "Component", + "spec": { + "type": "other", + "lifecycle": "unknown", + "owner": "unknown" + }, + "relations": [ + { + "target": { + "kind": "group", + "namespace": "default", + "name": "unknown" + }, + "type": "ownedBy" + } + ] + } + ] +} diff --git a/plugins/catalog-import/src/setupTests.ts b/plugins/catalog-import/src/setupTests.ts index 825bcd4115..aea2220869 100644 --- a/plugins/catalog-import/src/setupTests.ts +++ b/plugins/catalog-import/src/setupTests.ts @@ -15,3 +15,4 @@ */ import '@testing-library/jest-dom'; +import 'cross-fetch/polyfill'; diff --git a/plugins/catalog-import/src/util/useGithubRepos.ts b/plugins/catalog-import/src/util/useGithubRepos.ts index 89448a1be1..a2d299b9f4 100644 --- a/plugins/catalog-import/src/util/useGithubRepos.ts +++ b/plugins/catalog-import/src/util/useGithubRepos.ts @@ -22,18 +22,21 @@ import parseGitUrl from 'git-url-parse'; // TODO: (O5ten) Refactor into a core API instead of direct usage like this // https://github.com/backstage/backstage/pull/3613#issuecomment-7408929430 -import { readGitHubIntegrationConfigs } from '@backstage/integration'; +import { + GitHubIntegrationConfig, + readGitHubIntegrationConfigs, +} from '@backstage/integration'; export function useGithubRepos() { const api = useApi(catalogImportApiRef); const config = useApi(configApiRef); - const submitPrToRepo = async (selectedRepo: ConfigSpec) => { + const getGithubIntegrationConfig = (location: string) => { const { name: repoName, owner: ownerName, resource: hostname, - } = parseGitUrl(selectedRepo.location); + } = parseGitUrl(location); const configs = readGitHubIntegrationConfigs( config.getOptionalConfigArray('integrations.github') ?? [], @@ -41,9 +44,22 @@ export function useGithubRepos() { const githubIntegrationConfig = configs.find(v => v.host === hostname); if (!githubIntegrationConfig) { throw new Error( - `Unable to locate github-integration for repo-location: ${selectedRepo.location}`, + `Unable to locate github-integration for repo-location, ${location}`, ); } + return { + repoName, + ownerName, + githubIntegrationConfig, + }; + }; + + const submitPrToRepo = async (selectedRepo: ConfigSpec) => { + const { + repoName, + ownerName, + githubIntegrationConfig, + } = getGithubIntegrationConfig(selectedRepo.location); const submitPRResponse = await api .submitPrToRepo({ owner: ownerName, @@ -54,7 +70,7 @@ export function useGithubRepos() { githubIntegrationConfig, }) .catch(e => { - throw new Error(`Failed to submit PR to repo:\n${e.message}`); + throw new Error(`Failed to submit PR to repo, ${e.message}`); }); await api @@ -62,14 +78,41 @@ export function useGithubRepos() { location: submitPRResponse.location, }) .catch(e => { - throw new Error(`Failed to create repository location:\n${e.message}`); + throw new Error(`Failed to create repository location, ${e.message}`); }); return submitPRResponse; }; + const checkForExistingCatalogInfo = async ( + location: string, + ): Promise<{ exists: boolean; url?: string }> => { + let githubConfig: { + repoName: string; + ownerName: string; + githubIntegrationConfig: GitHubIntegrationConfig; + }; + try { + githubConfig = getGithubIntegrationConfig(location); + } catch (e) { + return { exists: false }; + } + return await api + .checkForExistingCatalogInfo({ + owner: githubConfig.ownerName, + repo: githubConfig.repoName, + githubIntegrationConfig: githubConfig.githubIntegrationConfig, + }) + .catch(e => { + throw new Error( + `Failed to inspect repository for existing catalog-info.yaml, ${e.message}`, + ); + }); + }; + return { submitPrToRepo, + checkForExistingCatalogInfo, generateEntityDefinitions: (repo: string) => api.generateEntityDefinitions({ repo }), addLocation: (location: string) => diff --git a/plugins/catalog/CHANGELOG.md b/plugins/catalog/CHANGELOG.md index b826346b39..efacf6fbad 100644 --- a/plugins/catalog/CHANGELOG.md +++ b/plugins/catalog/CHANGELOG.md @@ -1,5 +1,33 @@ # @backstage/plugin-catalog +## 0.2.13 + +### Patch Changes + +- a4e636c8f: Hide the kind of owners in the about card if it's the default kind (group) +- 099c5cf4f: Show the parent component in the about card (via partOf relationship) +- a08db734c: Remove the `WelcomeBanner` that links to a plugin that is not longer wired into Backstage instances + +## 0.2.12 + +### Patch Changes + +- 593632f07: Derive the list of to-delete entities in the `UnregisterEntityDialog` from the `backstage.io/managed-by-origin-location` annotation. + The dialog also rejects deleting entities that are created by the `bootstrap:bootstrap` location. +- 33846acfc: Display the owner, system, and domain as links to the entity pages in the about card. + Only display fields in the about card that are applicable to the entity kind. +- f04db53d7: Display systems in catalog table and make both owner and system link to the entity pages. + The owner field is now taken from the relations of the entity instead of its spec. +- Updated dependencies [def2307f3] +- Updated dependencies [efd6ef753] +- Updated dependencies [a187b8ad0] +- Updated dependencies [ed6baab66] +- Updated dependencies [a93f42213] + - @backstage/catalog-model@0.7.0 + - @backstage/core@0.5.0 + - @backstage/plugin-scaffolder@0.4.0 + - @backstage/catalog-client@0.3.5 + ## 0.2.11 ### Patch Changes diff --git a/plugins/catalog/package.json b/plugins/catalog/package.json index fa6d757f0a..be1681c596 100644 --- a/plugins/catalog/package.json +++ b/plugins/catalog/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-catalog", - "version": "0.2.11", + "version": "0.2.13", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -30,10 +30,10 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@backstage/catalog-client": "^0.3.4", - "@backstage/catalog-model": "^0.6.1", - "@backstage/core": "^0.4.4", - "@backstage/plugin-scaffolder": "^0.3.6", + "@backstage/catalog-client": "^0.3.5", + "@backstage/catalog-model": "^0.7.0", + "@backstage/core": "^0.5.0", + "@backstage/plugin-scaffolder": "^0.4.0", "@backstage/theme": "^0.2.2", "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", @@ -51,8 +51,8 @@ "swr": "^0.3.0" }, "devDependencies": { - "@backstage/cli": "^0.4.6", - "@backstage/dev-utils": "^0.1.7", + "@backstage/cli": "^0.4.7", + "@backstage/dev-utils": "^0.1.8", "@backstage/test-utils": "^0.1.6", "@microsoft/microsoft-graph-types": "^1.25.0", "@testing-library/jest-dom": "^5.10.1", diff --git a/plugins/catalog/src/components/AboutCard/AboutContent.tsx b/plugins/catalog/src/components/AboutCard/AboutContent.tsx index 178debb42a..b2d029c927 100644 --- a/plugins/catalog/src/components/AboutCard/AboutContent.tsx +++ b/plugins/catalog/src/components/AboutCard/AboutContent.tsx @@ -40,9 +40,17 @@ export const AboutContent = ({ entity }: Props) => { const isSystem = entity.kind.toLowerCase() === 'system'; const isDomain = entity.kind.toLowerCase() === 'domain'; const isResource = entity.kind.toLowerCase() === 'resource'; + const isComponent = entity.kind.toLowerCase() === 'component'; const [partOfSystemRelation] = getEntityRelations(entity, RELATION_PART_OF, { kind: 'system', }); + const [partOfComponentRelation] = getEntityRelations( + entity, + RELATION_PART_OF, + { + kind: 'component', + }, + ); const [partOfDomainRelation] = getEntityRelations(entity, RELATION_PART_OF, { kind: 'domain', }); @@ -59,7 +67,7 @@ export const AboutContent = ({ entity }: Props) => { {ownedByRelations.map((t, i) => ( {i > 0 && ', '} - + ))} @@ -91,6 +99,18 @@ export const AboutContent = ({ entity }: Props) => { )} )} + {isComponent && partOfComponentRelation && ( + + + + )} {!isSystem && !isDomain && ( { onChange={({ label }) => setSelectedTab(label)} /> -