diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index d53ca1b607..69bd8393c8 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -3,6 +3,8 @@ name: E2E Test Linux on: pull_request: paths-ignore: + - 'contrib/**' + - 'docs/**' - 'microsite/**' jobs: diff --git a/CHANGELOG.md b/CHANGELOG.md index 294c57815e..cf68fad6bd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,13 +6,15 @@ If you encounter issues while upgrading to a newer version, don't hesitate to re ## Next Release +> Collect changes for the next release below + +## v0.1.1-alpha.22 + ### @backstage/core - Introduced initial version of an inverted app/plugin relationship, where plugins export components for apps to use, instead registering themselves directly into the app. This enables more fine-grained control of plugin features, and also composition of plugins such as catalog pages with additional cards and tabs. This breaks the use of `RouteRef`s, and there will be more changes related to this in the future, but this change lays the initial foundation. See `packages/app` and followup PRs for how to update plugins for this change. [#2076](https://github.com/spotify/backstage/pull/2076) - Switch to an automatic dependency injection mechanism for all Utility APIs, allowing plugins to ship default implementations of their APIs. See [https://backstage.io/docs/api/utility-apis](https://backstage.io/docs/api/utility-apis). [#2285](https://github.com/spotify/backstage/pull/2285) -> Collect changes for the next release below - ### @backstage/cli - Change `backstage-cli backend:build-image` to forward all args to `docker image build`, instead of just tag. Also add `--build` flag for building all dependent packages before packaging the workspace for the docker build. [#2299](https://github.com/spotify/backstage/pull/2299) @@ -21,6 +23,10 @@ If you encounter issues while upgrading to a newer version, don't hesitate to re - Change root `tsc` output dir to `dist-types`, in order to allow for standalone plugin repos. [#2278](https://github.com/spotify/backstage/pull/2278) +### @backstage/catalog-backend + +- We have simplified the way that GitHub ingestion works. The `catalog.processors.githubApi` key is deprecated, in favor of `catalog.processors.github`. At the same time, the location type `github/api` is likewise deprecated, in favor of `github`. This location type now serves both raw HTTP reads and APIv3 reads, depending on how you configure it. It also supports having several providers at once - for example, both public GitHub and an internal GitHub Enterprise, with different keys. If you still use the `catalog.processors.githubApi` config key, things will work but you will get a deprecation warning at startup. In a later release, support for the old key will go away entirely. See the [configuration section in the docs](https://backstage.io/docs/features/software-catalog/configuration) for more details. + ## v0.1.1-alpha.21 - Added many more frontend plugins to the template along with the sidebar. [#1942](https://github.com/spotify/backstage/pull/1942), [#2084](https://github.com/spotify/backstage/pull/2084) diff --git a/app-config.yaml b/app-config.yaml index e018eef87e..60ecb813f7 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -35,6 +35,8 @@ organization: techdocs: storageUrl: http://localhost:7000/techdocs/static/docs requestUrl: http://localhost:7000/techdocs/docs + generators: + techdocs: 'docker' sentry: organization: spotify @@ -58,21 +60,23 @@ catalog: - allow: [Component, API, Group, Template, Location] processors: github: - privateToken: - $secret: - env: GITHUB_PRIVATE_TOKEN - githubApi: providers: - target: https://github.com token: $secret: env: GITHUB_PRIVATE_TOKEN - # Example for how to add your GitHub Enterprise instance: + #### Example for how to add your GitHub Enterprise instance using the API: # - target: https://ghe.example.net # apiBaseUrl: https://ghe.example.net/api/v3 # token: # $secret: # env: GHE_PRIVATE_TOKEN + #### Example for how to add your GitHub Enterprise instance using raw HTTP fetches (token is optional): + # - target: https://ghe.example.net + # rawBaseUrl: https://ghe.example.net/raw + # token: + # $secret: + # env: GHE_PRIVATE_TOKEN bitbucketApi: username: $secret: diff --git a/contrib/README.md b/contrib/README.md new file mode 100644 index 0000000000..2bf33c063b --- /dev/null +++ b/contrib/README.md @@ -0,0 +1,9 @@ +# Backstage Contrib + +This directory contains various community contributions related to Backstage. + +Unless otherwise specified, all content in this hierarchy fall under the same +[licensing terms](../LICENSE) as in the rest of the repository, and come with +no guarantees of functionality or fitness of purpose. That being said, we +really appreciate contributions in here and encourage them being kept up to +date. diff --git a/contrib/kubernetes/basic_kubernetes_example_with_helm/README.md b/contrib/kubernetes/basic_kubernetes_example_with_helm/README.md new file mode 100644 index 0000000000..2dc875f671 --- /dev/null +++ b/contrib/kubernetes/basic_kubernetes_example_with_helm/README.md @@ -0,0 +1 @@ +# Basic Kubernetes example with Helm diff --git a/install/kubernetes/app.yaml b/contrib/kubernetes/basic_kubernetes_example_with_helm/app.yaml similarity index 100% rename from install/kubernetes/app.yaml rename to contrib/kubernetes/basic_kubernetes_example_with_helm/app.yaml diff --git a/install/kubernetes/backend.yaml b/contrib/kubernetes/basic_kubernetes_example_with_helm/backend.yaml similarity index 100% rename from install/kubernetes/backend.yaml rename to contrib/kubernetes/basic_kubernetes_example_with_helm/backend.yaml diff --git a/install/kubernetes/backstage/.helmignore b/contrib/kubernetes/basic_kubernetes_example_with_helm/backstage/.helmignore similarity index 100% rename from install/kubernetes/backstage/.helmignore rename to contrib/kubernetes/basic_kubernetes_example_with_helm/backstage/.helmignore diff --git a/install/kubernetes/backstage/Chart.yaml b/contrib/kubernetes/basic_kubernetes_example_with_helm/backstage/Chart.yaml similarity index 100% rename from install/kubernetes/backstage/Chart.yaml rename to contrib/kubernetes/basic_kubernetes_example_with_helm/backstage/Chart.yaml diff --git a/install/kubernetes/backstage/README.md b/contrib/kubernetes/basic_kubernetes_example_with_helm/backstage/README.md similarity index 100% rename from install/kubernetes/backstage/README.md rename to contrib/kubernetes/basic_kubernetes_example_with_helm/backstage/README.md diff --git a/install/kubernetes/backstage/templates/_helpers.tpl b/contrib/kubernetes/basic_kubernetes_example_with_helm/backstage/templates/_helpers.tpl similarity index 100% rename from install/kubernetes/backstage/templates/_helpers.tpl rename to contrib/kubernetes/basic_kubernetes_example_with_helm/backstage/templates/_helpers.tpl diff --git a/install/kubernetes/backstage/templates/deployment.yaml b/contrib/kubernetes/basic_kubernetes_example_with_helm/backstage/templates/deployment.yaml similarity index 100% rename from install/kubernetes/backstage/templates/deployment.yaml rename to contrib/kubernetes/basic_kubernetes_example_with_helm/backstage/templates/deployment.yaml diff --git a/install/kubernetes/backstage/templates/ingress.yaml b/contrib/kubernetes/basic_kubernetes_example_with_helm/backstage/templates/ingress.yaml similarity index 100% rename from install/kubernetes/backstage/templates/ingress.yaml rename to contrib/kubernetes/basic_kubernetes_example_with_helm/backstage/templates/ingress.yaml diff --git a/install/kubernetes/backstage/templates/service.yaml b/contrib/kubernetes/basic_kubernetes_example_with_helm/backstage/templates/service.yaml similarity index 100% rename from install/kubernetes/backstage/templates/service.yaml rename to contrib/kubernetes/basic_kubernetes_example_with_helm/backstage/templates/service.yaml diff --git a/install/kubernetes/backstage/values.yaml b/contrib/kubernetes/basic_kubernetes_example_with_helm/backstage/values.yaml similarity index 100% rename from install/kubernetes/backstage/values.yaml rename to contrib/kubernetes/basic_kubernetes_example_with_helm/backstage/values.yaml diff --git a/install/kubernetes/ingress.yaml b/contrib/kubernetes/basic_kubernetes_example_with_helm/ingress.yaml similarity index 100% rename from install/kubernetes/ingress.yaml rename to contrib/kubernetes/basic_kubernetes_example_with_helm/ingress.yaml diff --git a/install/kubernetes/service.yaml b/contrib/kubernetes/basic_kubernetes_example_with_helm/service.yaml similarity index 100% rename from install/kubernetes/service.yaml rename to contrib/kubernetes/basic_kubernetes_example_with_helm/service.yaml diff --git a/docs/features/software-catalog/configuration.md b/docs/features/software-catalog/configuration.md index 0bcfaabe0a..c98bd73ca3 100644 --- a/docs/features/software-catalog/configuration.md +++ b/docs/features/software-catalog/configuration.md @@ -4,6 +4,67 @@ title: Catalog Configuration description: Documentation on Software Catalog Configuration --- +## Processors + +The catalog makes use of so called processors to perform all kinds of ingestion +tasks, such as reading raw entity data from a remote source, parsing it, +transforming it, and validating it. These processors are configured under the +`catalog.processors` key. + +### Processor: github + +The `github` processor is responsible for fetching entity data from files on +GitHub or GitHub Enterprise. The configuration for this processor lives under +`catalog.processors.github`. Example: + +```yaml +catalog: + processors: + github: + providers: + - target: https://github.com + token: + $secret: + env: GITHUB_PRIVATE_TOKEN + - target: https://ghe.example.net + apiBaseUrl: https://ghe.example.net/api/v3 + rawBaseUrl: https://ghe.example.net/raw + token: + $secret: + env: GHE_PRIVATE_TOKEN +``` + +The main subkey is `providers`, where you can list the various GitHub compatible +providers you want to be able to fetch data from. Each entry is a structure with +up to four elements: + +- `target` (required): The string prefix of the location target that you want to + match on, with no trailing slash. For GitHub, it should be exactly + `https://github.com`. +- `token` (optional): An authentication token as expected by GitHub. If + supplied, it will be passed along with all calls to this provider, both API + and raw. If it is not supplied, anonymous access will be used. +- `apiBaseUrl` (optional): If you want to communicate using the APIv3 method + with this provider, specify the base URL for its endpoint here, with no + trailing slash. Specifically when the target is github, you can leave it out + to be inferred automatically. For a GitHub Enterprise installation, it is + commonly at `https://api.` or `https:///api/v3`. +- `rawBaseUrl` (optional): If you want to communicate using the raw HTTP method + with this provider, specify the base URL for its endpoint here, with no + trailing slash. Specifically when the target is public GitHub, you can leave + it out to be inferred automatically. For a GitHub Enterprise installation, it + is commonly at `https://api.` or `https:///api/v3`. + +You need to supply either `apiBaseUrl` or `rawBaseUrl` or both (except for +public GitHub, for which we can infer them). The `apiBaseUrl` will always be +preferred over the other if a `token` is given, otherwise `rawBaseUrl` will be +preferred. + +If you do not supply a public GitHub provider, one will be added automatically, +silently at startup for convenience. So you only have to list it if you want to +supply a token for it - and if you do, you can also leave out the `apiBaseUrl` +and `rawBaseUrl` fields. + ## Static Location Configuration To enable declarative catalog setups, it is possible to add locations to the diff --git a/docs/features/techdocs/getting-started.md b/docs/features/techdocs/getting-started.md index d8c836c970..b9d215fd8a 100644 --- a/docs/features/techdocs/getting-started.md +++ b/docs/features/techdocs/getting-started.md @@ -71,6 +71,30 @@ building and publishing of your documentation, you want to change the `requestUrl` to point to your storage. In this case `storageUrl` is not required. +### Disable Docker in Docker situation (Optional) + +The TechDocs backend plugin runs a docker container with mkdocs to generate the +frontend of the docs from source files (Markdown). If you are deploying +Backstage using Docker, this will mean that your Backstage Docker container will +try to run another Docker container for TechDocs backend. + +To avoid this problem, we have a configuration available. You can set a value in +your `app-config.yaml` that tells the techdocs generator if it should run the +`local` mkdocs or run it from `docker`. This defaults to running as `docker` if +no config is provided. + +```yaml +techdocs: + generators: + techdocs: local +``` + +Setting `generators.techdocs` to `local` means you will have to make sure your +environment is compatible with techdocs. You will have to install the +`mkdocs-techdocs-container` and 'mkdocs' package from pip, as well as graphviz +and plantuml from your package manager. This has only been tested with python +3.7 and python 3.8. + ## Run Backstage locally Change folder to `/packages/backend` and run the diff --git a/docs/tutorials/quickstart-app-auth.md b/docs/tutorials/quickstart-app-auth.md index fea1d7085a..2615e37633 100644 --- a/docs/tutorials/quickstart-app-auth.md +++ b/docs/tutorials/quickstart-app-auth.md @@ -41,7 +41,7 @@ From the terminal: ℹ ｢wds｣: Project is running at http://localhost:3000/ ``` -Once the app compiles, a browser window should have popped with your stand alone +Once the app compiles, a browser window should have popped with your stand-alone application loaded at `localhost:3000`. This could take a couple minutes. ```zsh diff --git a/lerna.json b/lerna.json index 63e4701ce3..a018b1a289 100644 --- a/lerna.json +++ b/lerna.json @@ -2,5 +2,5 @@ "packages": ["packages/*", "plugins/*"], "npmClient": "yarn", "useWorkspaces": true, - "version": "0.1.1-alpha.20" + "version": "0.1.1-alpha.22" } diff --git a/microsite/sidebars.json b/microsite/sidebars.json index aac524c538..8fa4935a38 100644 --- a/microsite/sidebars.json +++ b/microsite/sidebars.json @@ -40,6 +40,7 @@ "ids": [ "features/software-catalog/software-catalog-overview", "features/software-catalog/installation", + "features/software-catalog/configuration", "features/software-catalog/system-model", "features/software-catalog/descriptor-format", "features/software-catalog/well-known-annotations", diff --git a/mkdocs.yml b/mkdocs.yml index a74ef5737f..ba607f8a6e 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -28,10 +28,11 @@ nav: - Features: - Software Catalog: - Overview: 'features/software-catalog/index.md' + - Installation: 'features/software-catalog/installation.md' + - Configuration: 'features/software-catalog/configuration.md' - System model: 'features/software-catalog/system-model.md' - YAML File Format: 'features/software-catalog/descriptor-format.md' - Well-known Annotations: 'features/software-catalog/well-known-annotations.md' - - Configuration: 'features/software-catalog/configuration.md' - Extending the model: 'features/software-catalog/extending-the-model.md' - External integrations: 'features/software-catalog/external-integrations.md' - API: 'features/software-catalog/api.md' diff --git a/packages/app/package.json b/packages/app/package.json index bf0f340a40..cb61e1ed8c 100644 --- a/packages/app/package.json +++ b/packages/app/package.json @@ -1,32 +1,32 @@ { "name": "example-app", - "version": "0.1.1-alpha.21", + "version": "0.1.1-alpha.22", "private": true, "bundled": true, "dependencies": { - "@backstage/cli": "^0.1.1-alpha.21", - "@backstage/catalog-model": "^0.1.1-alpha.21", - "@backstage/core": "^0.1.1-alpha.21", - "@backstage/plugin-api-docs": "^0.1.1-alpha.21", - "@backstage/plugin-catalog": "^0.1.1-alpha.21", - "@backstage/plugin-circleci": "^0.1.1-alpha.21", - "@backstage/plugin-explore": "^0.1.1-alpha.21", - "@backstage/plugin-gcp-projects": "^0.1.1-alpha.21", - "@backstage/plugin-github-actions": "^0.1.1-alpha.21", - "@backstage/plugin-gitops-profiles": "^0.1.1-alpha.21", - "@backstage/plugin-graphiql": "^0.1.1-alpha.21", - "@backstage/plugin-jenkins": "^0.1.1-alpha.21", - "@backstage/plugin-lighthouse": "^0.1.1-alpha.21", - "@backstage/plugin-newrelic": "^0.1.1-alpha.21", - "@backstage/plugin-register-component": "^0.1.1-alpha.21", - "@backstage/plugin-rollbar": "^0.1.1-alpha.21", - "@backstage/plugin-scaffolder": "^0.1.1-alpha.21", - "@backstage/plugin-sentry": "^0.1.1-alpha.21", - "@backstage/plugin-tech-radar": "^0.1.1-alpha.21", - "@backstage/plugin-techdocs": "^0.1.1-alpha.21", - "@backstage/plugin-welcome": "^0.1.1-alpha.21", - "@backstage/test-utils": "^0.1.1-alpha.21", - "@backstage/theme": "^0.1.1-alpha.21", + "@backstage/catalog-model": "^0.1.1-alpha.22", + "@backstage/cli": "^0.1.1-alpha.22", + "@backstage/core": "^0.1.1-alpha.22", + "@backstage/plugin-api-docs": "^0.1.1-alpha.22", + "@backstage/plugin-catalog": "^0.1.1-alpha.22", + "@backstage/plugin-circleci": "^0.1.1-alpha.22", + "@backstage/plugin-explore": "^0.1.1-alpha.22", + "@backstage/plugin-gcp-projects": "^0.1.1-alpha.22", + "@backstage/plugin-github-actions": "^0.1.1-alpha.22", + "@backstage/plugin-gitops-profiles": "^0.1.1-alpha.22", + "@backstage/plugin-graphiql": "^0.1.1-alpha.22", + "@backstage/plugin-jenkins": "^0.1.1-alpha.22", + "@backstage/plugin-lighthouse": "^0.1.1-alpha.22", + "@backstage/plugin-newrelic": "^0.1.1-alpha.22", + "@backstage/plugin-register-component": "^0.1.1-alpha.22", + "@backstage/plugin-rollbar": "^0.1.1-alpha.22", + "@backstage/plugin-scaffolder": "^0.1.1-alpha.22", + "@backstage/plugin-sentry": "^0.1.1-alpha.22", + "@backstage/plugin-tech-radar": "^0.1.1-alpha.22", + "@backstage/plugin-techdocs": "^0.1.1-alpha.22", + "@backstage/plugin-welcome": "^0.1.1-alpha.22", + "@backstage/test-utils": "^0.1.1-alpha.22", + "@backstage/theme": "^0.1.1-alpha.22", "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", "@octokit/rest": "^18.0.0", diff --git a/packages/backend-common/package.json b/packages/backend-common/package.json index c9103375eb..9d8a7061e0 100644 --- a/packages/backend-common/package.json +++ b/packages/backend-common/package.json @@ -1,7 +1,7 @@ { "name": "@backstage/backend-common", "description": "Common functionality library for Backstage backends", - "version": "0.1.1-alpha.21", + "version": "0.1.1-alpha.22", "main": "src/index.ts", "types": "src/index.ts", "private": false, @@ -29,9 +29,9 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@backstage/cli-common": "^0.1.1-alpha.21", - "@backstage/config": "^0.1.1-alpha.21", - "@backstage/config-loader": "^0.1.1-alpha.21", + "@backstage/cli-common": "^0.1.1-alpha.22", + "@backstage/config": "^0.1.1-alpha.22", + "@backstage/config-loader": "^0.1.1-alpha.22", "@types/cors": "^2.8.6", "@types/express": "^4.17.6", "compression": "^1.7.4", @@ -42,12 +42,12 @@ "helmet": "^4.0.0", "knex": "^0.21.1", "lodash": "^4.17.15", + "logform": "^2.1.1", "morgan": "^1.10.0", "prom-client": "^12.0.0", "selfsigned": "^1.10.7", "stoppable": "^1.1.0", - "winston": "^3.2.1", - "logform": "^2.1.1" + "winston": "^3.2.1" }, "peerDependencies": { "pg-connection-string": "^2.3.0" @@ -58,7 +58,7 @@ } }, "devDependencies": { - "@backstage/cli": "^0.1.1-alpha.21", + "@backstage/cli": "^0.1.1-alpha.22", "@types/compression": "^1.7.0", "@types/http-errors": "^1.6.3", "@types/morgan": "^1.9.0", diff --git a/packages/backend/package.json b/packages/backend/package.json index c255f65015..aa6f0ad0bf 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -1,6 +1,6 @@ { "name": "example-backend", - "version": "0.1.1-alpha.21", + "version": "0.1.1-alpha.22", "main": "dist/index.cjs.js", "types": "src/index.ts", "private": true, @@ -18,23 +18,23 @@ "migrate:create": "knex migrate:make -x ts" }, "dependencies": { - "@backstage/backend-common": "^0.1.1-alpha.21", - "@backstage/catalog-model": "^0.1.1-alpha.21", - "@backstage/config": "^0.1.1-alpha.21", - "@backstage/plugin-app-backend": "^0.1.1-alpha.21", - "@backstage/plugin-auth-backend": "^0.1.1-alpha.21", - "@backstage/plugin-catalog-backend": "^0.1.1-alpha.21", - "@backstage/plugin-graphql-backend": "^0.1.1-alpha.21", - "@backstage/plugin-identity-backend": "^0.1.1-alpha.21", - "@backstage/plugin-proxy-backend": "^0.1.1-alpha.21", - "@backstage/plugin-rollbar-backend": "^0.1.1-alpha.21", - "@backstage/plugin-scaffolder-backend": "^0.1.1-alpha.21", - "@backstage/plugin-sentry-backend": "^0.1.1-alpha.21", - "@backstage/plugin-techdocs-backend": "^0.1.1-alpha.21", + "@backstage/backend-common": "^0.1.1-alpha.22", + "@backstage/catalog-model": "^0.1.1-alpha.22", + "@backstage/config": "^0.1.1-alpha.22", + "@backstage/plugin-app-backend": "^0.1.1-alpha.22", + "@backstage/plugin-auth-backend": "^0.1.1-alpha.22", + "@backstage/plugin-catalog-backend": "^0.1.1-alpha.22", + "@backstage/plugin-graphql-backend": "^0.1.1-alpha.22", + "@backstage/plugin-identity-backend": "^0.1.1-alpha.22", + "@backstage/plugin-proxy-backend": "^0.1.1-alpha.22", + "@backstage/plugin-rollbar-backend": "^0.1.1-alpha.22", + "@backstage/plugin-scaffolder-backend": "^0.1.1-alpha.22", + "@backstage/plugin-sentry-backend": "^0.1.1-alpha.22", + "@backstage/plugin-techdocs-backend": "^0.1.1-alpha.22", "@gitbeaker/node": "^23.5.0", "@octokit/rest": "^18.0.0", "dockerode": "^3.2.0", - "example-app": "^0.1.1-alpha.21", + "example-app": "^0.1.1-alpha.22", "express": "^4.17.1", "knex": "^0.21.1", "pg": "^8.3.0", @@ -43,7 +43,7 @@ "winston": "^3.2.1" }, "devDependencies": { - "@backstage/cli": "^0.1.1-alpha.21", + "@backstage/cli": "^0.1.1-alpha.22", "@types/dockerode": "^2.5.32", "@types/express": "^4.17.6", "@types/express-serve-static-core": "^4.17.5", diff --git a/packages/backend/src/plugins/techdocs.ts b/packages/backend/src/plugins/techdocs.ts index 7364cc4d83..58dca83b43 100644 --- a/packages/backend/src/plugins/techdocs.ts +++ b/packages/backend/src/plugins/techdocs.ts @@ -31,7 +31,7 @@ export default async function createPlugin({ config, }: PluginEnvironment) { const generators = new Generators(); - const techdocsGenerator = new TechdocsGenerator(logger); + const techdocsGenerator = new TechdocsGenerator(logger, config); generators.register('techdocs', techdocsGenerator); const preparers = new Preparers(); diff --git a/packages/catalog-model/package.json b/packages/catalog-model/package.json index 0d18125d43..d683eec68d 100644 --- a/packages/catalog-model/package.json +++ b/packages/catalog-model/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/catalog-model", - "version": "0.1.1-alpha.21", + "version": "0.1.1-alpha.22", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -20,7 +20,7 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@backstage/config": "^0.1.1-alpha.21", + "@backstage/config": "^0.1.1-alpha.22", "@types/json-schema": "^7.0.5", "@types/yup": "^0.28.2", "json-schema": "^0.2.5", @@ -29,7 +29,7 @@ "yup": "^0.29.1" }, "devDependencies": { - "@backstage/cli": "^0.1.1-alpha.21", + "@backstage/cli": "^0.1.1-alpha.22", "@types/express": "^4.17.6", "@types/jest": "^26.0.7", "@types/lodash": "^4.14.151", diff --git a/packages/cli-common/package.json b/packages/cli-common/package.json index 4132907e70..1340b46ae3 100644 --- a/packages/cli-common/package.json +++ b/packages/cli-common/package.json @@ -1,7 +1,7 @@ { "name": "@backstage/cli-common", "description": "Common functionality used by cli, backend, and create-app", - "version": "0.1.1-alpha.21", + "version": "0.1.1-alpha.22", "private": false, "main": "src/index.ts", "types": "src/index.ts", diff --git a/packages/cli/package.json b/packages/cli/package.json index 492cd3b958..e1f51be5b7 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,7 +1,7 @@ { "name": "@backstage/cli", "description": "CLI for developing Backstage plugins and apps", - "version": "0.1.1-alpha.21", + "version": "0.1.1-alpha.22", "private": false, "publishConfig": { "access": "public" @@ -28,9 +28,9 @@ "backstage-cli": "bin/backstage-cli" }, "dependencies": { - "@backstage/cli-common": "^0.1.1-alpha.21", - "@backstage/config": "^0.1.1-alpha.21", - "@backstage/config-loader": "^0.1.1-alpha.21", + "@backstage/cli-common": "^0.1.1-alpha.22", + "@backstage/config": "^0.1.1-alpha.22", + "@backstage/config-loader": "^0.1.1-alpha.22", "@hot-loader/react-dom": "^16.13.0", "@lerna/package-graph": "^3.18.5", "@lerna/project": "^3.18.0", diff --git a/packages/config-loader/package.json b/packages/config-loader/package.json index 592b8eefb3..5960b553f7 100644 --- a/packages/config-loader/package.json +++ b/packages/config-loader/package.json @@ -1,7 +1,7 @@ { "name": "@backstage/config-loader", "description": "Config loading functionality used by Backstage backend, and CLI", - "version": "0.1.1-alpha.21", + "version": "0.1.1-alpha.22", "private": false, "publishConfig": { "access": "public", @@ -30,7 +30,7 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@backstage/config": "^0.1.1-alpha.21", + "@backstage/config": "^0.1.1-alpha.22", "fs-extra": "^9.0.0", "yaml": "^1.9.2", "yup": "^0.29.1" diff --git a/packages/config/package.json b/packages/config/package.json index 236964f2bd..0b61e50ac6 100644 --- a/packages/config/package.json +++ b/packages/config/package.json @@ -1,7 +1,7 @@ { "name": "@backstage/config", "description": "Config API used by Backstage core, backend, and CLI", - "version": "0.1.1-alpha.21", + "version": "0.1.1-alpha.22", "private": false, "publishConfig": { "access": "public", diff --git a/packages/core-api/package.json b/packages/core-api/package.json index 191394e536..72d0748183 100644 --- a/packages/core-api/package.json +++ b/packages/core-api/package.json @@ -1,7 +1,7 @@ { "name": "@backstage/core-api", "description": "Internal Core API used by Backstage plugins and apps", - "version": "0.1.1-alpha.21", + "version": "0.1.1-alpha.22", "private": false, "publishConfig": { "access": "public", @@ -29,8 +29,8 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@backstage/config": "^0.1.1-alpha.21", - "@backstage/theme": "^0.1.1-alpha.21", + "@backstage/config": "^0.1.1-alpha.22", + "@backstage/theme": "^0.1.1-alpha.22", "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", "@types/react": "^16.9", @@ -41,8 +41,8 @@ "zen-observable": "^0.8.15" }, "devDependencies": { - "@backstage/cli": "^0.1.1-alpha.21", - "@backstage/test-utils-core": "^0.1.1-alpha.21", + "@backstage/cli": "^0.1.1-alpha.22", + "@backstage/test-utils-core": "^0.1.1-alpha.22", "@testing-library/jest-dom": "^5.10.1", "@testing-library/react": "^10.4.1", "@testing-library/user-event": "^12.0.7", diff --git a/packages/core/package.json b/packages/core/package.json index 7e9932bdc1..3ed730e0f3 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -1,7 +1,7 @@ { "name": "@backstage/core", "description": "Core API used by Backstage plugins and apps", - "version": "0.1.1-alpha.21", + "version": "0.1.1-alpha.22", "private": false, "publishConfig": { "access": "public", @@ -29,9 +29,9 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@backstage/config": "^0.1.1-alpha.21", - "@backstage/core-api": "^0.1.1-alpha.21", - "@backstage/theme": "^0.1.1-alpha.21", + "@backstage/config": "^0.1.1-alpha.22", + "@backstage/core-api": "^0.1.1-alpha.22", + "@backstage/theme": "^0.1.1-alpha.22", "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", "@material-ui/lab": "4.0.0-alpha.45", @@ -54,8 +54,8 @@ "react-use": "^15.3.3" }, "devDependencies": { - "@backstage/cli": "^0.1.1-alpha.21", - "@backstage/test-utils": "^0.1.1-alpha.21", + "@backstage/cli": "^0.1.1-alpha.22", + "@backstage/test-utils": "^0.1.1-alpha.22", "@testing-library/jest-dom": "^5.10.1", "@testing-library/react": "^10.4.1", "@testing-library/user-event": "^12.0.7", diff --git a/packages/create-app/package.json b/packages/create-app/package.json index af7b78eb74..b303c2423b 100644 --- a/packages/create-app/package.json +++ b/packages/create-app/package.json @@ -1,7 +1,7 @@ { "name": "@backstage/create-app", "description": "Create app package for Backstage", - "version": "0.1.1-alpha.21", + "version": "0.1.1-alpha.22", "private": false, "publishConfig": { "access": "public" @@ -27,7 +27,7 @@ "start": "nodemon --" }, "dependencies": { - "@backstage/cli-common": "^0.1.1-alpha.21", + "@backstage/cli-common": "^0.1.1-alpha.22", "chalk": "^4.0.0", "commander": "^6.1.0", "fs-extra": "^9.0.0", diff --git a/packages/create-app/templates/default-app/app-config.yaml.hbs b/packages/create-app/templates/default-app/app-config.yaml.hbs index ecac0613d9..a7d7aa8010 100644 --- a/packages/create-app/templates/default-app/app-config.yaml.hbs +++ b/packages/create-app/templates/default-app/app-config.yaml.hbs @@ -46,6 +46,8 @@ proxy: techdocs: storageUrl: http://localhost:7000/techdocs/static/docs requestUrl: http://localhost:7000/techdocs/docs + generators: + techdocs: 'docker' lighthouse: baseUrl: http://localhost:3003 @@ -65,10 +67,6 @@ catalog: - allow: [Component, API, Group, Template, Location] processors: github: - privateToken: - $secret: - env: GITHUB_PRIVATE_TOKEN - githubApi: providers: - target: https://github.com token: diff --git a/packages/create-app/templates/default-app/packages/backend/src/plugins/techdocs.ts b/packages/create-app/templates/default-app/packages/backend/src/plugins/techdocs.ts index dfd9eb5e33..9c7de3512b 100644 --- a/packages/create-app/templates/default-app/packages/backend/src/plugins/techdocs.ts +++ b/packages/create-app/templates/default-app/packages/backend/src/plugins/techdocs.ts @@ -15,7 +15,7 @@ export default async function createPlugin({ config, }: PluginEnvironment) { const generators = new Generators(); - const techdocsGenerator = new TechdocsGenerator(logger); + const techdocsGenerator = new TechdocsGenerator(logger, config); generators.register('techdocs', techdocsGenerator); diff --git a/packages/dev-utils/package.json b/packages/dev-utils/package.json index fe7272bf04..0b6e04f322 100644 --- a/packages/dev-utils/package.json +++ b/packages/dev-utils/package.json @@ -1,7 +1,7 @@ { "name": "@backstage/dev-utils", "description": "Utilities for developing Backstage plugins.", - "version": "0.1.1-alpha.21", + "version": "0.1.1-alpha.22", "private": false, "publishConfig": { "access": "public", @@ -29,10 +29,10 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@backstage/cli": "^0.1.1-alpha.21", - "@backstage/core": "^0.1.1-alpha.21", - "@backstage/test-utils": "^0.1.1-alpha.21", - "@backstage/theme": "^0.1.1-alpha.21", + "@backstage/cli": "^0.1.1-alpha.22", + "@backstage/core": "^0.1.1-alpha.22", + "@backstage/test-utils": "^0.1.1-alpha.22", + "@backstage/theme": "^0.1.1-alpha.22", "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", "@testing-library/jest-dom": "^5.10.1", diff --git a/packages/docgen/package.json b/packages/docgen/package.json index c75ea53faf..3836af6542 100644 --- a/packages/docgen/package.json +++ b/packages/docgen/package.json @@ -1,7 +1,7 @@ { "name": "docgen", "description": "Tool for generating API Documentation for itself", - "version": "0.1.1-alpha.21", + "version": "0.1.1-alpha.22", "private": true, "homepage": "https://backstage.io", "repository": { diff --git a/packages/e2e-test/package.json b/packages/e2e-test/package.json index 0af233ec8e..e518d4f710 100644 --- a/packages/e2e-test/package.json +++ b/packages/e2e-test/package.json @@ -1,7 +1,7 @@ { "name": "e2e-test", "description": "E2E test for verifying Backstage packages", - "version": "0.1.1-alpha.21", + "version": "0.1.1-alpha.22", "private": true, "homepage": "https://backstage.io", "repository": { @@ -21,7 +21,7 @@ "test:e2e": "yarn start" }, "devDependencies": { - "@backstage/cli-common": "^0.1.1-alpha.21", + "@backstage/cli-common": "^0.1.1-alpha.22", "@types/fs-extra": "^9.0.1", "@types/node": "^13.7.2", "fs-extra": "^9.0.0", diff --git a/packages/storybook/package.json b/packages/storybook/package.json index 8f12a2891c..a1decf279a 100644 --- a/packages/storybook/package.json +++ b/packages/storybook/package.json @@ -1,6 +1,6 @@ { "name": "storybook", - "version": "0.1.1-alpha.21", + "version": "0.1.1-alpha.22", "description": "Storybook build for core package", "private": true, "scripts": { @@ -14,7 +14,7 @@ ] }, "dependencies": { - "@backstage/theme": "^0.1.1-alpha.21" + "@backstage/theme": "^0.1.1-alpha.22" }, "devDependencies": { "@storybook/addon-actions": "^6.0.21", diff --git a/packages/techdocs-cli/package.json b/packages/techdocs-cli/package.json index 222620f976..340d9e2aff 100644 --- a/packages/techdocs-cli/package.json +++ b/packages/techdocs-cli/package.json @@ -1,7 +1,7 @@ { "name": "@techdocs/cli", "description": "CLI for running TechDocs locally.", - "version": "0.1.1-alpha.21", + "version": "0.1.1-alpha.22", "private": false, "publishConfig": { "access": "public" @@ -44,7 +44,7 @@ "ext": "ts" }, "dependencies": { - "@backstage/cli": "^0.1.1-alpha.21", + "@backstage/cli": "^0.1.1-alpha.22", "commander": "^6.1.0", "fs-extra": "^9.0.1", "http-proxy": "^1.18.1", diff --git a/packages/test-utils-core/package.json b/packages/test-utils-core/package.json index 8439bee779..d79acf80a8 100644 --- a/packages/test-utils-core/package.json +++ b/packages/test-utils-core/package.json @@ -1,7 +1,7 @@ { "name": "@backstage/test-utils-core", "description": "Utilities to test Backstage core", - "version": "0.1.1-alpha.21", + "version": "0.1.1-alpha.22", "private": false, "publishConfig": { "access": "public", diff --git a/packages/test-utils/package.json b/packages/test-utils/package.json index 8567af8c14..cb115fe633 100644 --- a/packages/test-utils/package.json +++ b/packages/test-utils/package.json @@ -1,7 +1,7 @@ { "name": "@backstage/test-utils", "description": "Utilities to test Backstage plugins and apps.", - "version": "0.1.1-alpha.21", + "version": "0.1.1-alpha.22", "private": false, "publishConfig": { "access": "public", @@ -29,10 +29,10 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@backstage/cli": "^0.1.1-alpha.21", - "@backstage/core-api": "^0.1.1-alpha.21", - "@backstage/test-utils-core": "^0.1.1-alpha.21", - "@backstage/theme": "^0.1.1-alpha.21", + "@backstage/cli": "^0.1.1-alpha.22", + "@backstage/core-api": "^0.1.1-alpha.22", + "@backstage/test-utils-core": "^0.1.1-alpha.22", + "@backstage/theme": "^0.1.1-alpha.22", "@material-ui/core": "^4.11.0", "@testing-library/jest-dom": "^5.10.1", "@testing-library/react": "^10.4.1", diff --git a/packages/theme/package.json b/packages/theme/package.json index 436455633b..a1a3d6d111 100644 --- a/packages/theme/package.json +++ b/packages/theme/package.json @@ -1,7 +1,7 @@ { "name": "@backstage/theme", "description": "material-ui theme for use with Backstage.", - "version": "0.1.1-alpha.21", + "version": "0.1.1-alpha.22", "private": false, "publishConfig": { "access": "public", @@ -31,7 +31,7 @@ "@material-ui/core": "^4.11.0" }, "devDependencies": { - "@backstage/cli": "^0.1.1-alpha.21" + "@backstage/cli": "^0.1.1-alpha.22" }, "files": [ "dist" diff --git a/plugins/api-docs/package.json b/plugins/api-docs/package.json index 25e5c6b900..378ceb2d08 100644 --- a/plugins/api-docs/package.json +++ b/plugins/api-docs/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-api-docs", - "version": "0.1.1-alpha.21", + "version": "0.1.1-alpha.22", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -20,10 +20,10 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@backstage/catalog-model": "^0.1.1-alpha.21", - "@backstage/core": "^0.1.1-alpha.21", - "@backstage/plugin-catalog": "^0.1.1-alpha.21", - "@backstage/theme": "^0.1.1-alpha.21", + "@backstage/catalog-model": "^0.1.1-alpha.22", + "@backstage/core": "^0.1.1-alpha.22", + "@backstage/plugin-catalog": "^0.1.1-alpha.22", + "@backstage/theme": "^0.1.1-alpha.22", "@kyma-project/asyncapi-react": "^0.11.0", "@material-icons/font": "^1.0.2", "@material-ui/core": "^4.11.0", @@ -39,9 +39,9 @@ "swagger-ui-react": "^3.31.1" }, "devDependencies": { - "@backstage/cli": "^0.1.1-alpha.21", - "@backstage/dev-utils": "^0.1.1-alpha.21", - "@backstage/test-utils": "^0.1.1-alpha.21", + "@backstage/cli": "^0.1.1-alpha.22", + "@backstage/dev-utils": "^0.1.1-alpha.22", + "@backstage/test-utils": "^0.1.1-alpha.22", "@testing-library/jest-dom": "^5.10.1", "@testing-library/react": "^10.4.1", "@testing-library/user-event": "^12.0.7", diff --git a/plugins/app-backend/package.json b/plugins/app-backend/package.json index f4080abc42..c4738c7105 100644 --- a/plugins/app-backend/package.json +++ b/plugins/app-backend/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-app-backend", - "version": "0.1.1-alpha.21", + "version": "0.1.1-alpha.22", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -20,8 +20,8 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@backstage/backend-common": "^0.1.1-alpha.21", - "@backstage/config-loader": "^0.1.1-alpha.21", + "@backstage/backend-common": "^0.1.1-alpha.22", + "@backstage/config-loader": "^0.1.1-alpha.22", "@types/express": "^4.17.6", "express": "^4.17.1", "express-promise-router": "^3.0.3", @@ -30,7 +30,7 @@ "yn": "^4.0.0" }, "devDependencies": { - "@backstage/cli": "^0.1.1-alpha.21", + "@backstage/cli": "^0.1.1-alpha.22", "@types/supertest": "^2.0.8", "msw": "^0.19.5", "supertest": "^4.0.2" diff --git a/plugins/auth-backend/package.json b/plugins/auth-backend/package.json index 2336adb860..91fe08c0fa 100644 --- a/plugins/auth-backend/package.json +++ b/plugins/auth-backend/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-auth-backend", - "version": "0.1.1-alpha.21", + "version": "0.1.1-alpha.22", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -20,8 +20,8 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@backstage/backend-common": "^0.1.1-alpha.21", - "@backstage/config": "^0.1.1-alpha.21", + "@backstage/backend-common": "^0.1.1-alpha.22", + "@backstage/config": "^0.1.1-alpha.22", "@types/express": "^4.17.6", "compression": "^1.7.4", "cookie-parser": "^1.4.5", @@ -49,7 +49,7 @@ "yn": "^4.0.0" }, "devDependencies": { - "@backstage/cli": "^0.1.1-alpha.21", + "@backstage/cli": "^0.1.1-alpha.22", "@types/body-parser": "^1.19.0", "@types/cookie-parser": "^1.4.2", "@types/jwt-decode": "2.2.1", diff --git a/plugins/catalog-backend/package.json b/plugins/catalog-backend/package.json index d1711cf3d5..f443ff362f 100644 --- a/plugins/catalog-backend/package.json +++ b/plugins/catalog-backend/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-catalog-backend", - "version": "0.1.1-alpha.21", + "version": "0.1.1-alpha.22", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -20,13 +20,14 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@backstage/backend-common": "^0.1.1-alpha.21", - "@backstage/catalog-model": "^0.1.1-alpha.21", - "@backstage/config": "^0.1.1-alpha.21", + "@backstage/backend-common": "^0.1.1-alpha.22", + "@backstage/catalog-model": "^0.1.1-alpha.22", + "@backstage/config": "^0.1.1-alpha.22", "@types/express": "^4.17.6", "express": "^4.17.1", "express-promise-router": "^3.0.3", "fs-extra": "^9.0.0", + "git-url-parse": "^11.2.0", "knex": "^0.21.1", "lodash": "^4.17.15", "morgan": "^1.10.0", @@ -39,7 +40,8 @@ "yup": "^0.29.1" }, "devDependencies": { - "@backstage/cli": "^0.1.1-alpha.21", + "@backstage/cli": "^0.1.1-alpha.22", + "@types/git-url-parse": "^9.0.0", "@types/lodash": "^4.14.151", "@types/node-fetch": "^2.5.7", "@types/supertest": "^2.0.8", diff --git a/plugins/catalog-backend/src/database/CommonDatabase.ts b/plugins/catalog-backend/src/database/CommonDatabase.ts index 0c56ca4370..4ad5362fb6 100644 --- a/plugins/catalog-backend/src/database/CommonDatabase.ts +++ b/plugins/catalog-backend/src/database/CommonDatabase.ts @@ -343,7 +343,14 @@ export class CommonDatabase implements Database { entityName?: string, message?: string, ): Promise { - return this.database( + // Remove log entries older than a day + const cutoff = new Date(); + cutoff.setDate(cutoff.getDate() - 1); + await this.database('location_update_log') + .where('created_at', '<', cutoff.toISOString()) + .del(); + + await this.database( 'location_update_log', ).insert({ status, diff --git a/plugins/catalog-backend/src/ingestion/LocationReaders.ts b/plugins/catalog-backend/src/ingestion/LocationReaders.ts index 6f3a27f00d..8da6e20ab3 100644 --- a/plugins/catalog-backend/src/ingestion/LocationReaders.ts +++ b/plugins/catalog-backend/src/ingestion/LocationReaders.ts @@ -27,7 +27,6 @@ import { AnnotateLocationEntityProcessor } from './processors/AnnotateLocationEn import { EntityPolicyProcessor } from './processors/EntityPolicyProcessor'; import { FileReaderProcessor } from './processors/FileReaderProcessor'; import { GithubReaderProcessor } from './processors/GithubReaderProcessor'; -import { GithubApiReaderProcessor } from './processors/GithubApiReaderProcessor'; import { GitlabApiReaderProcessor } from './processors/GitlabApiReaderProcessor'; import { GitlabReaderProcessor } from './processors/GitlabReaderProcessor'; import { BitbucketApiReaderProcessor } from './processors/BitbucketApiReaderProcessor'; @@ -67,18 +66,19 @@ export class LocationReaders implements LocationReader { private readonly rulesEnforcer: CatalogRulesEnforcer; static defaultProcessors(options: { + logger: Logger; config?: Config; entityPolicy?: EntityPolicy; }): LocationProcessor[] { const { + logger, config = new ConfigReader({}, 'missing-config'), entityPolicy = new EntityPolicies(), } = options; return [ StaticLocationProcessor.fromConfig(config), new FileReaderProcessor(), - new GithubReaderProcessor(config), - GithubApiReaderProcessor.fromConfig(config), + GithubReaderProcessor.fromConfig(config, logger), new GitlabApiReaderProcessor(config), new GitlabReaderProcessor(), new BitbucketApiReaderProcessor(config), @@ -94,7 +94,7 @@ export class LocationReaders implements LocationReader { constructor({ logger = getVoidLogger(), config, - processors = LocationReaders.defaultProcessors({ config }), + processors = LocationReaders.defaultProcessors({ logger, config }), }: Options) { this.logger = logger; this.processors = processors; diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.test.ts deleted file mode 100644 index 48f38f0e97..0000000000 --- a/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.test.ts +++ /dev/null @@ -1,162 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { LocationSpec } from '@backstage/catalog-model'; -import { ConfigReader } from '@backstage/config'; -import { - getRawUrl, - getRequestOptions, - GithubApiReaderProcessor, - ProviderConfig, - readConfig, -} from './GithubApiReaderProcessor'; - -describe('GithubApiReaderProcessor', () => { - describe('getRequestOptions', () => { - it('sets the correct API version', () => { - const config: ProviderConfig = { target: '', apiBaseUrl: '' }; - expect((getRequestOptions(config).headers as any).Accept).toEqual( - 'application/vnd.github.v3.raw', - ); - }); - - it('inserts a token when needed', () => { - const withToken: ProviderConfig = { - target: '', - apiBaseUrl: '', - token: 'A', - }; - const withoutToken: ProviderConfig = { - target: '', - apiBaseUrl: '', - }; - expect( - (getRequestOptions(withToken).headers as any).Authorization, - ).toEqual('token A'); - expect( - (getRequestOptions(withoutToken).headers as any).Authorization, - ).toBeUndefined(); - }); - }); - - describe('getRawUrl', () => { - it('rejects targets that do not look like URLs', () => { - const config: ProviderConfig = { target: '', apiBaseUrl: '' }; - expect(() => getRawUrl('a/b', config)).toThrow(/Incorrect URL: a\/b/); - }); - - it('passes through the happy path', () => { - const config: ProviderConfig = { - target: 'https://github.com', - apiBaseUrl: 'https://api.github.com', - }; - expect( - getRawUrl( - 'https://github.com/a/b/blob/branchname/path/to/c.yaml', - config, - ), - ).toEqual( - new URL( - 'https://api.github.com/repos/a/b/contents/path/to/c.yaml?ref=branchname', - ), - ); - }); - }); - - describe('readConfig', () => { - function config( - providers: { target: string; apiBaseUrl?: string; token?: string }[], - ) { - return ConfigReader.fromConfigs([ - { - context: '', - data: { - catalog: { processors: { githubApi: { providers } } }, - }, - }, - ]); - } - - it('adds a default GitHub entry when missing', () => { - const output = readConfig(config([])); - expect(output).toEqual([ - { target: 'https://github.com', apiBaseUrl: 'https://api.github.com' }, - ]); - }); - - it('injects the correct GitHub API base URL when missing', () => { - const output = readConfig(config([{ target: 'https://github.com' }])); - expect(output).toEqual([ - { target: 'https://github.com', apiBaseUrl: 'https://api.github.com' }, - ]); - }); - - it('rejects custom targets with no API base URL', () => { - expect(() => - readConfig(config([{ target: 'https://ghe.company.com' }])), - ).toThrow( - 'Provider at https://ghe.company.com must configure an explicit apiBaseUrl', - ); - }); - - it('rejects funky configs', () => { - expect(() => readConfig(config([{ target: 7 } as any]))).toThrow( - /target/, - ); - expect(() => readConfig(config([{ noTarget: '7' } as any]))).toThrow( - /target/, - ); - expect(() => - readConfig( - config([{ target: 'https://github.com', apiBaseUrl: 7 } as any]), - ), - ).toThrow(/apiBaseUrl/); - expect(() => - readConfig(config([{ target: 'https://github.com', token: 7 } as any])), - ).toThrow(/token/); - }); - }); - - describe('implementation', () => { - it('rejects unknown types', async () => { - const processor = new GithubApiReaderProcessor([ - { target: 'https://github.com', apiBaseUrl: 'https://api.github.com' }, - ]); - const location: LocationSpec = { - type: 'not-github/api', - target: 'https://github.com', - }; - await expect( - processor.readLocation(location, false, () => {}), - ).resolves.toBeFalsy(); - }); - - it('rejects unknown targets', async () => { - const processor = new GithubApiReaderProcessor([ - { target: 'https://github.com', apiBaseUrl: 'https://api.github.com' }, - ]); - const location: LocationSpec = { - type: 'github/api', - target: 'https://not.github.com/apa', - }; - await expect( - processor.readLocation(location, false, () => {}), - ).rejects.toThrow( - /There is no GitHub provider that matches https:\/\/not.github.com\/apa/, - ); - }); - }); -}); diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.ts deleted file mode 100644 index 812bf6f488..0000000000 --- a/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.ts +++ /dev/null @@ -1,192 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { LocationSpec } from '@backstage/catalog-model'; -import { Config } from '@backstage/config'; -import fetch, { HeadersInit, RequestInit } from 'node-fetch'; -import * as result from './results'; -import { LocationProcessor, LocationProcessorEmit } from './types'; - -/** - * The configuration parameters for a single GitHub API provider. - */ -export type ProviderConfig = { - /** - * The prefix of the target that this matches on, e.g. "https://github.com", - * with no trailing slash. - */ - target: string; - - /** - * The base URL of the API of this provider, e.g. "https://api.github.com", - * with no trailing slash. - */ - apiBaseUrl: string; - - /** - * The authorization token to use for requests to this provider. - * - * If no token is specified, anonymous API access is used. - */ - token?: string; -}; - -export function getRequestOptions(provider: ProviderConfig): RequestInit { - const headers: HeadersInit = { - Accept: 'application/vnd.github.v3.raw', - }; - - if (provider.token) { - headers.Authorization = `token ${provider.token}`; - } - - return { - headers, - }; -} - -// Converts for example -// from: https://github.com/a/b/blob/branchname/path/to/c.yaml -// to: https://api.github.com/repos/a/b/contents/path/to/c.yaml?ref=branchname -export function getRawUrl(target: string, provider: ProviderConfig): URL { - try { - const oldPath = new URL(target).pathname.split('/'); - const [, userOrOrg, repoName, blobOrRaw, ref, ...restOfPath] = oldPath; - - if ( - !userOrOrg || - !repoName || - (blobOrRaw !== 'blob' && blobOrRaw !== 'raw') || - !restOfPath.join('/').match(/\.ya?ml$/) - ) { - throw new Error('Wrong URL or Invalid file path'); - } - - // Transform to API path - const newPath = [ - 'repos', - userOrOrg, - repoName, - 'contents', - ...restOfPath, - ].join('/'); - return new URL(`${provider.apiBaseUrl}/${newPath}?ref=${ref}`); - } catch (e) { - throw new Error(`Incorrect URL: ${target}, ${e}`); - } -} - -export function readConfig(configRoot: Config): ProviderConfig[] { - const providers: ProviderConfig[] = []; - - // In a previous version of the configuration, we only supported github, - // and the "privateToken" key held the token to use for it. The new - // configuration method is to use the "providers" key instead. - const config = configRoot.getOptionalConfig('catalog.processors.githubApi'); - const providerConfigs = config?.getOptionalConfigArray('providers') ?? []; - const legacyToken = config?.getOptionalString('privateToken'); - - // First read all the explicit providers - for (const providerConfig of providerConfigs) { - const target = providerConfig.getString('target').replace(/\/+$/, ''); - let apiBaseUrl = providerConfig.getOptionalString('apiBaseUrl'); - const token = providerConfig.getOptionalString('token'); - - if (apiBaseUrl) { - apiBaseUrl = apiBaseUrl.replace(/\/+$/, ''); - } else if (target === 'https://github.com') { - apiBaseUrl = 'https://api.github.com'; - } else { - throw new Error( - `Provider at ${target} must configure an explicit apiBaseUrl`, - ); - } - - providers.push({ target, apiBaseUrl, token }); - } - - // If no explicit github.com provider was added, put one in the list as - // a convenience - if (!providers.some(p => p.target === 'https://github.com')) { - providers.push({ - target: 'https://github.com', - apiBaseUrl: 'https://api.github.com', - token: legacyToken, - }); - } - - return providers; -} - -/** - * A processor that adds the ability to read files from GitHub v3 APIs, such as - * the one exposed by GitHub itself. - */ -export class GithubApiReaderProcessor implements LocationProcessor { - private providers: ProviderConfig[]; - - static fromConfig(config: Config) { - return new GithubApiReaderProcessor(readConfig(config)); - } - - constructor(providers: ProviderConfig[]) { - this.providers = providers; - } - - async readLocation( - location: LocationSpec, - optional: boolean, - emit: LocationProcessorEmit, - ): Promise { - if (location.type !== 'github/api') { - return false; - } - - const provider = this.providers.find(p => - location.target.startsWith(`${p.target}/`), - ); - if (!provider) { - throw new Error( - `There is no GitHub provider that matches ${location.target}. Please add a configuration entry for it under catalog.github.processors.githubApi.`, - ); - } - - try { - const url = getRawUrl(location.target, provider); - const options = getRequestOptions(provider); - const response = await fetch(url.toString(), options); - - if (response.ok) { - const data = await response.buffer(); - emit(result.data(location, data)); - } else { - const message = `${location.target} could not be read as ${url}, ${response.status} ${response.statusText}`; - if (response.status === 404) { - if (!optional) { - emit(result.notFoundError(location, message)); - } - } else { - emit(result.generalError(location, message)); - } - } - } catch (e) { - const message = `Unable to read ${location.type} ${location.target}, ${e}`; - emit(result.generalError(location, message)); - } - - return true; - } -} diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.test.ts new file mode 100644 index 0000000000..3ab24541e7 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.test.ts @@ -0,0 +1,269 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { getVoidLogger } from '@backstage/backend-common'; +import { LocationSpec } from '@backstage/catalog-model'; +import { ConfigReader } from '@backstage/config'; +import { + getApiRequestOptions, + getApiUrl, + getRawRequestOptions, + getRawUrl, + GithubReaderProcessor, + ProviderConfig, + readConfig, +} from './GithubReaderProcessor'; + +describe('GithubReaderProcessor', () => { + describe('getApiRequestOptions', () => { + it('sets the correct API version', () => { + const config: ProviderConfig = { target: '', apiBaseUrl: '' }; + expect((getApiRequestOptions(config).headers as any).Accept).toEqual( + 'application/vnd.github.v3.raw', + ); + }); + + it('inserts a token when needed', () => { + const withToken: ProviderConfig = { + target: '', + apiBaseUrl: '', + token: 'A', + }; + const withoutToken: ProviderConfig = { + target: '', + apiBaseUrl: '', + }; + expect( + (getApiRequestOptions(withToken).headers as any).Authorization, + ).toEqual('token A'); + expect( + (getApiRequestOptions(withoutToken).headers as any).Authorization, + ).toBeUndefined(); + }); + }); + + describe('getRawRequestOptions', () => { + it('inserts a token when needed', () => { + const withToken: ProviderConfig = { + target: '', + rawBaseUrl: '', + token: 'A', + }; + const withoutToken: ProviderConfig = { + target: '', + rawBaseUrl: '', + }; + expect( + (getRawRequestOptions(withToken).headers as any).Authorization, + ).toEqual('token A'); + expect( + (getRawRequestOptions(withoutToken).headers as any).Authorization, + ).toBeUndefined(); + }); + }); + + describe('getApiUrl', () => { + it('rejects targets that do not look like URLs', () => { + const config: ProviderConfig = { target: '', apiBaseUrl: '' }; + expect(() => getApiUrl('a/b', config)).toThrow(/Incorrect URL: a\/b/); + }); + + it('happy path for github', () => { + const config: ProviderConfig = { + target: 'https://github.com', + apiBaseUrl: 'https://api.github.com', + }; + expect( + getApiUrl( + 'https://github.com/a/b/blob/branchname/path/to/c.yaml', + config, + ), + ).toEqual( + new URL( + 'https://api.github.com/repos/a/b/contents/path/to/c.yaml?ref=branchname', + ), + ); + expect( + getApiUrl( + 'https://ghe.mycompany.net/a/b/blob/branchname/path/to/c.yaml', + config, + ), + ).toEqual( + new URL( + 'https://api.github.com/repos/a/b/contents/path/to/c.yaml?ref=branchname', + ), + ); + }); + + it('happy path for ghe', () => { + const config: ProviderConfig = { + target: 'https://ghe.mycompany.net', + apiBaseUrl: 'https://ghe.mycompany.net/api/v3', + }; + expect( + getApiUrl( + 'https://ghe.mycompany.net/a/b/blob/branchname/path/to/c.yaml', + config, + ), + ).toEqual( + new URL( + 'https://ghe.mycompany.net/api/v3/repos/a/b/contents/path/to/c.yaml?ref=branchname', + ), + ); + }); + }); + + describe('getRawUrl', () => { + it('rejects targets that do not look like URLs', () => { + const config: ProviderConfig = { target: '', apiBaseUrl: '' }; + expect(() => getRawUrl('a/b', config)).toThrow(/Incorrect URL: a\/b/); + }); + + it('happy path for github', () => { + const config: ProviderConfig = { + target: 'https://github.com', + rawBaseUrl: 'https://raw.githubusercontent.com', + }; + expect( + getRawUrl( + 'https://github.com/a/b/blob/branchname/path/to/c.yaml', + config, + ), + ).toEqual( + new URL( + 'https://raw.githubusercontent.com/a/b/branchname/path/to/c.yaml', + ), + ); + }); + + it('happy path for ghe', () => { + const config: ProviderConfig = { + target: 'https://ghe.mycompany.net', + rawBaseUrl: 'https://ghe.mycompany.net/raw', + }; + expect( + getRawUrl( + 'https://ghe.mycompany.net/a/b/blob/branchname/path/to/c.yaml', + config, + ), + ).toEqual( + new URL('https://ghe.mycompany.net/raw/a/b/branchname/path/to/c.yaml'), + ); + }); + }); + + describe('readConfig', () => { + function config( + providers: { target: string; apiBaseUrl?: string; token?: string }[], + ) { + return ConfigReader.fromConfigs([ + { + context: '', + data: { + catalog: { processors: { github: { providers } } }, + }, + }, + ]); + } + + it('adds a default GitHub entry when missing', () => { + const output = readConfig(config([]), getVoidLogger()); + expect(output).toEqual([ + { + target: 'https://github.com', + apiBaseUrl: 'https://api.github.com', + rawBaseUrl: 'https://raw.githubusercontent.com', + }, + ]); + }); + + it('injects the correct GitHub API base URL when missing', () => { + const output = readConfig( + config([{ target: 'https://github.com' }]), + getVoidLogger(), + ); + expect(output).toEqual([ + { + target: 'https://github.com', + apiBaseUrl: 'https://api.github.com', + rawBaseUrl: 'https://raw.githubusercontent.com', + }, + ]); + }); + + it('rejects custom targets with no base URLs', () => { + expect(() => + readConfig( + config([{ target: 'https://ghe.company.com' }]), + getVoidLogger(), + ), + ).toThrow( + 'Provider at https://ghe.company.com must configure an explicit apiBaseUrl or rawBaseUrl', + ); + }); + + it('rejects funky configs', () => { + expect(() => + readConfig(config([{ target: 7 } as any]), getVoidLogger()), + ).toThrow(/target/); + expect(() => + readConfig(config([{ noTarget: '7' } as any]), getVoidLogger()), + ).toThrow(/target/); + expect(() => + readConfig( + config([{ target: 'https://github.com', apiBaseUrl: 7 } as any]), + getVoidLogger(), + ), + ).toThrow(/apiBaseUrl/); + expect(() => + readConfig( + config([{ target: 'https://github.com', token: 7 } as any]), + getVoidLogger(), + ), + ).toThrow(/token/); + }); + }); + + describe('implementation', () => { + it('rejects unknown types', async () => { + const processor = new GithubReaderProcessor([ + { target: 'https://github.com', apiBaseUrl: 'https://api.github.com' }, + ]); + const location: LocationSpec = { + type: 'not-github/api', + target: 'https://github.com', + }; + await expect( + processor.readLocation(location, false, () => {}), + ).resolves.toBeFalsy(); + }); + + it('rejects unknown targets', async () => { + const processor = new GithubReaderProcessor([ + { target: 'https://github.com', apiBaseUrl: 'https://api.github.com' }, + ]); + const location: LocationSpec = { + type: 'github/api', + target: 'https://not.github.com/apa', + }; + await expect( + processor.readLocation(location, false, () => {}), + ).rejects.toThrow( + /There is no GitHub provider that matches https:\/\/not.github.com\/apa/, + ); + }); + }); +}); diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.ts index 9f38d782fe..4f0c66148f 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.ts @@ -15,33 +15,207 @@ */ import { LocationSpec } from '@backstage/catalog-model'; -import fetch, { RequestInit, HeadersInit } from 'node-fetch'; +import { Config } from '@backstage/config'; +import parseGitUri from 'git-url-parse'; +import fetch, { HeadersInit, RequestInit } from 'node-fetch'; +import { Logger } from 'winston'; import * as result from './results'; import { LocationProcessor, LocationProcessorEmit } from './types'; -import { Config } from '@backstage/config'; -export class GithubReaderProcessor implements LocationProcessor { - private privateToken: string; +/** + * The configuration parameters for a single GitHub API provider. + */ +export type ProviderConfig = { + /** + * The prefix of the target that this matches on, e.g. "https://github.com", + * with no trailing slash. + */ + target: string; - constructor(config?: Config) { - this.privateToken = - config?.getOptionalString('catalog.processors.github.privateToken') ?? ''; + /** + * The base URL of the API of this provider, e.g. "https://api.github.com", + * with no trailing slash. + * + * May be omitted specifically for GitHub; then it will be deduced. + * + * The API will always be preferred if both its base URL and a token are + * present. + */ + apiBaseUrl?: string; + + /** + * The base URL of the raw fetch endpoint of this provider, e.g. + * "https://raw.githubusercontent.com", with no trailing slash. + * + * May be omitted specifically for GitHub; then it will be deduced. + * + * The API will always be preferred if both its base URL and a token are + * present. + */ + rawBaseUrl?: string; + + /** + * The authorization token to use for requests to this provider. + * + * If no token is specified, anonymous access is used. + */ + token?: string; +}; + +export function getApiRequestOptions(provider: ProviderConfig): RequestInit { + const headers: HeadersInit = { + Accept: 'application/vnd.github.v3.raw', + }; + + if (provider.token) { + headers.Authorization = `token ${provider.token}`; } - getRequestOptions(): RequestInit { - const headers: HeadersInit = { - Accept: 'application/vnd.github.v3.raw', - }; + return { + headers, + }; +} - if (this.privateToken !== '') { - headers.Authorization = `token ${this.privateToken}`; +export function getRawRequestOptions(provider: ProviderConfig): RequestInit { + const headers: HeadersInit = {}; + + if (provider.token) { + headers.Authorization = `token ${provider.token}`; + } + + return { + headers, + }; +} + +// Converts for example +// from: https://github.com/a/b/blob/branchname/path/to/c.yaml +// to: https://api.github.com/repos/a/b/contents/path/to/c.yaml?ref=branchname +export function getApiUrl(target: string, provider: ProviderConfig): URL { + try { + const { owner, name, ref, filepathtype, filepath } = parseGitUri(target); + + if ( + !owner || + !name || + !ref || + (filepathtype !== 'blob' && filepathtype !== 'raw') || + !filepath?.match(/\.ya?ml$/) + ) { + throw new Error('Wrong URL or invalid file path'); } - const requestOptions: RequestInit = { - headers, - }; + const pathWithoutSlash = filepath.replace(/^\//, ''); + return new URL( + `${provider.apiBaseUrl}/repos/${owner}/${name}/contents/${pathWithoutSlash}?ref=${ref}`, + ); + } catch (e) { + throw new Error(`Incorrect URL: ${target}, ${e}`); + } +} - return requestOptions; +// Converts for example +// from: https://github.com/a/b/blob/branchname/c.yaml +// to: https://raw.githubusercontent.com/a/b/branchname/c.yaml +export function getRawUrl(target: string, provider: ProviderConfig): URL { + try { + const { owner, name, ref, filepathtype, filepath } = parseGitUri(target); + + if ( + !owner || + !name || + !ref || + (filepathtype !== 'blob' && filepathtype !== 'raw') || + !filepath?.match(/\.ya?ml$/) + ) { + throw new Error('Wrong URL or invalid file path'); + } + + const pathWithoutSlash = filepath.replace(/^\//, ''); + return new URL( + `${provider.rawBaseUrl}/${owner}/${name}/${ref}/${pathWithoutSlash}`, + ); + } catch (e) { + throw new Error(`Incorrect URL: ${target}, ${e}`); + } +} + +export function readConfig(config: Config, logger: Logger): ProviderConfig[] { + const providers: ProviderConfig[] = []; + + // TODO(freben): Deprecate the old config root entirely in a later release + if (config.has('catalog.processors.githubApi')) { + logger.warn( + 'The catalog.processors.githubApi configuration key has been deprecated, please use catalog.processors.github instead', + ); + } + + // In a previous version of the configuration, we only supported github, + // and the "privateToken" key held the token to use for it. The new + // configuration method is to use the "providers" key instead. + const providerConfigs = + config.getOptionalConfigArray('catalog.processors.github.providers') ?? + config.getOptionalConfigArray('catalog.processors.githubApi.providers') ?? + []; + const legacyToken = + config.getOptionalString('catalog.processors.github.privateToken') ?? + config.getOptionalString('catalog.processors.githubApi.privateToken'); + + // First read all the explicit providers + for (const providerConfig of providerConfigs) { + const target = providerConfig.getString('target').replace(/\/+$/, ''); + let apiBaseUrl = providerConfig.getOptionalString('apiBaseUrl'); + let rawBaseUrl = providerConfig.getOptionalString('rawBaseUrl'); + const token = providerConfig.getOptionalString('token'); + + if (apiBaseUrl) { + apiBaseUrl = apiBaseUrl.replace(/\/+$/, ''); + } else if (target === 'https://github.com') { + apiBaseUrl = 'https://api.github.com'; + } + + if (rawBaseUrl) { + rawBaseUrl = rawBaseUrl.replace(/\/+$/, ''); + } else if (target === 'https://github.com') { + rawBaseUrl = 'https://raw.githubusercontent.com'; + } + + if (!apiBaseUrl && !rawBaseUrl) { + throw new Error( + `Provider at ${target} must configure an explicit apiBaseUrl or rawBaseUrl`, + ); + } + + providers.push({ target, apiBaseUrl, rawBaseUrl, token }); + } + + // If no explicit github.com provider was added, put one in the list as + // a convenience + if (!providers.some(p => p.target === 'https://github.com')) { + providers.push({ + target: 'https://github.com', + apiBaseUrl: 'https://api.github.com', + rawBaseUrl: 'https://raw.githubusercontent.com', + token: legacyToken, + }); + } + + return providers; +} + +/** + * A processor that adds the ability to read files from GitHub v3 APIs, such as + * the one exposed by GitHub itself. + */ +export class GithubReaderProcessor implements LocationProcessor { + private providers: ProviderConfig[]; + + static fromConfig(config: Config, logger: Logger) { + return new GithubReaderProcessor(readConfig(config, logger)); + } + + constructor(providers: ProviderConfig[]) { + this.providers = providers; } async readLocation( @@ -49,16 +223,30 @@ export class GithubReaderProcessor implements LocationProcessor { optional: boolean, emit: LocationProcessorEmit, ): Promise { - if (location.type !== 'github') { + // The github/api type is for backward compatibility + if (location.type !== 'github' && location.type !== 'github/api') { return false; } - try { - const url = this.buildRawUrl(location.target); + const provider = this.providers.find(p => + location.target.startsWith(`${p.target}/`), + ); + if (!provider) { + throw new Error( + `There is no GitHub provider that matches ${location.target}. Please add a configuration entry for it under catalog.processors.github.providers.`, + ); + } - // TODO(freben): Should "hard" errors thrown by this line be treated as - // notFound instead of fatal? - const response = await fetch(url.toString(), this.getRequestOptions()); + try { + const useApi = + provider.apiBaseUrl && (provider.token || !provider.rawBaseUrl); + const url = useApi + ? getApiUrl(location.target, provider) + : getRawUrl(location.target, provider); + const options = useApi + ? getApiRequestOptions(provider) + : getRawRequestOptions(provider); + const response = await fetch(url.toString(), options); if (response.ok) { const data = await response.buffer(); @@ -80,41 +268,4 @@ export class GithubReaderProcessor implements LocationProcessor { return true; } - - // Converts - // from: https://github.com/a/b/blob/master/c.yaml - // to: https://raw.githubusercontent.com/a/b/master/c.yaml - private buildRawUrl(target: string): URL { - try { - const url = new URL(target); - - const [ - empty, - userOrOrg, - repoName, - blobKeyword, - ...restOfPath - ] = url.pathname.split('/'); - - if ( - url.hostname !== 'github.com' || - empty !== '' || - userOrOrg === '' || - repoName === '' || - blobKeyword !== 'blob' || - !restOfPath.join('/').match(/\.yaml$/) - ) { - throw new Error('Wrong GitHub URL'); - } - - // Removing the "blob" part - url.pathname = [empty, userOrOrg, repoName, ...restOfPath].join('/'); - url.hostname = 'raw.githubusercontent.com'; - url.protocol = 'https'; - - return url; - } catch (e) { - throw new Error(`Incorrect url: ${target}, ${e}`); - } - } } diff --git a/plugins/catalog-graphql/package.json b/plugins/catalog-graphql/package.json index 5a3eef4cce..c508215f29 100644 --- a/plugins/catalog-graphql/package.json +++ b/plugins/catalog-graphql/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-catalog-graphql", - "version": "0.1.1-alpha.21", + "version": "0.1.1-alpha.22", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -20,9 +20,9 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@backstage/catalog-model": "^0.1.1-alpha.18", - "@backstage/backend-common": "^0.1.1-alpha.18", - "@backstage/config": "^0.1.1-alpha.18", + "@backstage/catalog-model": "^0.1.1-alpha.22", + "@backstage/backend-common": "^0.1.1-alpha.22", + "@backstage/config": "^0.1.1-alpha.22", "@graphql-modules/core": "^0.7.17", "apollo-server": "^2.16.1", "graphql": "^15.3.0", @@ -32,7 +32,7 @@ "winston": "^3.2.1" }, "devDependencies": { - "@backstage/cli": "^0.1.1-alpha.18", + "@backstage/cli": "^0.1.1-alpha.22", "@types/express": "^4.17.7", "@types/supertest": "^2.0.8", "@graphql-codegen/cli": "^1.17.7", diff --git a/plugins/catalog/package.json b/plugins/catalog/package.json index e1b52795d3..f5520f39f9 100644 --- a/plugins/catalog/package.json +++ b/plugins/catalog/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-catalog", - "version": "0.1.1-alpha.21", + "version": "0.1.1-alpha.22", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -21,14 +21,15 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@backstage/catalog-model": "^0.1.1-alpha.21", - "@backstage/core": "^0.1.1-alpha.21", - "@backstage/plugin-scaffolder": "^0.1.1-alpha.21", - "@backstage/plugin-techdocs": "^0.1.1-alpha.21", - "@backstage/theme": "^0.1.1-alpha.21", + "@backstage/catalog-model": "^0.1.1-alpha.22", + "@backstage/core": "^0.1.1-alpha.22", + "@backstage/plugin-scaffolder": "^0.1.1-alpha.22", + "@backstage/plugin-techdocs": "^0.1.1-alpha.22", + "@backstage/theme": "^0.1.1-alpha.22", "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", "@material-ui/lab": "4.0.0-alpha.45", + "@types/react": "^16.9", "classnames": "^2.2.6", "moment": "^2.26.0", "react": "^16.13.1", @@ -37,13 +38,12 @@ "react-router": "6.0.0-beta.0", "react-router-dom": "6.0.0-beta.0", "react-use": "^15.3.3", - "swr": "^0.3.0", - "@types/react": "^16.9" + "swr": "^0.3.0" }, "devDependencies": { - "@backstage/cli": "^0.1.1-alpha.21", - "@backstage/dev-utils": "^0.1.1-alpha.21", - "@backstage/test-utils": "^0.1.1-alpha.21", + "@backstage/cli": "^0.1.1-alpha.22", + "@backstage/dev-utils": "^0.1.1-alpha.22", + "@backstage/test-utils": "^0.1.1-alpha.22", "@testing-library/jest-dom": "^5.10.1", "@testing-library/react": "^10.4.1", "@testing-library/react-hooks": "^3.3.0", diff --git a/plugins/circleci/package.json b/plugins/circleci/package.json index 7c8f36d370..10278fc065 100644 --- a/plugins/circleci/package.json +++ b/plugins/circleci/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-circleci", - "version": "0.1.1-alpha.21", + "version": "0.1.1-alpha.22", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -21,10 +21,10 @@ "postpack": "backstage-cli postpack" }, "dependencies": { - "@backstage/core": "^0.1.1-alpha.21", - "@backstage/catalog-model": "^0.1.1-alpha.21", - "@backstage/plugin-catalog": "^0.1.1-alpha.21", - "@backstage/theme": "^0.1.1-alpha.21", + "@backstage/catalog-model": "^0.1.1-alpha.22", + "@backstage/core": "^0.1.1-alpha.22", + "@backstage/plugin-catalog": "^0.1.1-alpha.22", + "@backstage/theme": "^0.1.1-alpha.22", "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", "@material-ui/lab": "4.0.0-alpha.45", @@ -38,8 +38,8 @@ "react-use": "^15.3.3" }, "devDependencies": { - "@backstage/cli": "^0.1.1-alpha.21", - "@backstage/dev-utils": "^0.1.1-alpha.21", + "@backstage/cli": "^0.1.1-alpha.22", + "@backstage/dev-utils": "^0.1.1-alpha.22", "@testing-library/jest-dom": "^5.10.1", "@testing-library/react": "^10.4.1", "@testing-library/user-event": "^12.0.7", diff --git a/plugins/explore/package.json b/plugins/explore/package.json index f302f4080c..c889f96153 100644 --- a/plugins/explore/package.json +++ b/plugins/explore/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-explore", - "version": "0.1.1-alpha.21", + "version": "0.1.1-alpha.22", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -21,21 +21,21 @@ "start": "backstage-cli plugin:serve" }, "dependencies": { - "@backstage/core": "^0.1.1-alpha.21", - "@backstage/theme": "^0.1.1-alpha.21", + "@backstage/core": "^0.1.1-alpha.22", + "@backstage/theme": "^0.1.1-alpha.22", "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", "@material-ui/lab": "4.0.0-alpha.45", "classnames": "^2.2.6", "react": "^16.13.1", "react-dom": "^16.13.1", - "react-use": "^15.3.3", - "react-router": "6.0.0-beta.0" + "react-router": "6.0.0-beta.0", + "react-use": "^15.3.3" }, "devDependencies": { - "@backstage/cli": "^0.1.1-alpha.21", - "@backstage/dev-utils": "^0.1.1-alpha.21", - "@backstage/test-utils": "^0.1.1-alpha.21", + "@backstage/cli": "^0.1.1-alpha.22", + "@backstage/dev-utils": "^0.1.1-alpha.22", + "@backstage/test-utils": "^0.1.1-alpha.22", "@testing-library/jest-dom": "^5.10.1", "@testing-library/react": "^10.4.1", "@testing-library/user-event": "^12.0.7", diff --git a/plugins/gcp-projects/package.json b/plugins/gcp-projects/package.json index cdc4bb507d..b71cebf6e4 100644 --- a/plugins/gcp-projects/package.json +++ b/plugins/gcp-projects/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-gcp-projects", - "version": "0.1.1-alpha.21", + "version": "0.1.1-alpha.22", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -21,8 +21,8 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@backstage/core": "^0.1.1-alpha.21", - "@backstage/theme": "^0.1.1-alpha.21", + "@backstage/core": "^0.1.1-alpha.22", + "@backstage/theme": "^0.1.1-alpha.22", "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", "@material-ui/lab": "4.0.0-alpha.45", @@ -32,8 +32,8 @@ "react-use": "^15.3.3" }, "devDependencies": { - "@backstage/cli": "^0.1.1-alpha.21", - "@backstage/dev-utils": "^0.1.1-alpha.21", + "@backstage/cli": "^0.1.1-alpha.22", + "@backstage/dev-utils": "^0.1.1-alpha.22", "@testing-library/jest-dom": "^5.10.1", "@testing-library/react": "^10.4.1", "@testing-library/user-event": "^12.0.7", diff --git a/plugins/github-actions/package.json b/plugins/github-actions/package.json index 49862dea80..9d1538ce99 100644 --- a/plugins/github-actions/package.json +++ b/plugins/github-actions/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-github-actions", - "version": "0.1.1-alpha.21", + "version": "0.1.1-alpha.22", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -21,11 +21,11 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@backstage/catalog-model": "^0.1.1-alpha.21", - "@backstage/core": "^0.1.1-alpha.21", - "@backstage/core-api": "^0.1.1-alpha.21", - "@backstage/plugin-catalog": "^0.1.1-alpha.21", - "@backstage/theme": "^0.1.1-alpha.21", + "@backstage/catalog-model": "^0.1.1-alpha.22", + "@backstage/core": "^0.1.1-alpha.22", + "@backstage/core-api": "^0.1.1-alpha.22", + "@backstage/plugin-catalog": "^0.1.1-alpha.22", + "@backstage/theme": "^0.1.1-alpha.22", "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", "@material-ui/lab": "4.0.0-alpha.45", @@ -40,8 +40,8 @@ "react-use": "^15.3.3" }, "devDependencies": { - "@backstage/cli": "^0.1.1-alpha.21", - "@backstage/dev-utils": "^0.1.1-alpha.21", + "@backstage/cli": "^0.1.1-alpha.22", + "@backstage/dev-utils": "^0.1.1-alpha.22", "@testing-library/jest-dom": "^5.10.1", "@testing-library/react": "^10.4.1", "@testing-library/user-event": "^12.0.7", diff --git a/plugins/gitops-profiles/package.json b/plugins/gitops-profiles/package.json index 62dacb5e91..cb25167ba6 100644 --- a/plugins/gitops-profiles/package.json +++ b/plugins/gitops-profiles/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-gitops-profiles", - "version": "0.1.1-alpha.21", + "version": "0.1.1-alpha.22", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -21,8 +21,8 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@backstage/core": "^0.1.1-alpha.21", - "@backstage/theme": "^0.1.1-alpha.21", + "@backstage/core": "^0.1.1-alpha.22", + "@backstage/theme": "^0.1.1-alpha.22", "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", "@material-ui/lab": "4.0.0-alpha.45", @@ -32,8 +32,8 @@ "react-use": "^15.3.3" }, "devDependencies": { - "@backstage/cli": "^0.1.1-alpha.21", - "@backstage/dev-utils": "^0.1.1-alpha.21", + "@backstage/cli": "^0.1.1-alpha.22", + "@backstage/dev-utils": "^0.1.1-alpha.22", "@testing-library/jest-dom": "^5.10.1", "@testing-library/react": "^10.4.1", "@testing-library/user-event": "^12.0.7", diff --git a/plugins/graphiql/package.json b/plugins/graphiql/package.json index b4d0b685da..e6122eb38b 100644 --- a/plugins/graphiql/package.json +++ b/plugins/graphiql/package.json @@ -1,7 +1,7 @@ { "name": "@backstage/plugin-graphiql", "description": "Backstage plugin for browsing GraphQL APIs", - "version": "0.1.1-alpha.21", + "version": "0.1.1-alpha.22", "private": false, "publishConfig": { "access": "public", @@ -31,8 +31,8 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@backstage/core": "^0.1.1-alpha.21", - "@backstage/theme": "^0.1.1-alpha.21", + "@backstage/core": "^0.1.1-alpha.22", + "@backstage/theme": "^0.1.1-alpha.22", "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", "@material-ui/lab": "4.0.0-alpha.45", @@ -43,9 +43,9 @@ "react-use": "^15.3.3" }, "devDependencies": { - "@backstage/cli": "^0.1.1-alpha.21", - "@backstage/dev-utils": "^0.1.1-alpha.21", - "@backstage/test-utils": "^0.1.1-alpha.21", + "@backstage/cli": "^0.1.1-alpha.22", + "@backstage/dev-utils": "^0.1.1-alpha.22", + "@backstage/test-utils": "^0.1.1-alpha.22", "@testing-library/jest-dom": "^5.10.1", "@testing-library/react": "^10.4.1", "@testing-library/user-event": "^12.0.7", diff --git a/plugins/graphql/package.json b/plugins/graphql/package.json index 9ad0d7c156..9ba972bd22 100644 --- a/plugins/graphql/package.json +++ b/plugins/graphql/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-graphql-backend", - "version": "0.1.1-alpha.21", + "version": "0.1.1-alpha.22", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -19,10 +19,10 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@backstage/backend-common": "^0.1.1-alpha.21", - "@backstage/config": "^0.1.1-alpha.21", - "@backstage/plugin-catalog-graphql": "^0.1.1-alpha.21", + "@backstage/config": "^0.1.1-alpha.22", + "@backstage/plugin-catalog-graphql": "^0.1.1-alpha.22", "@graphql-modules/core": "^0.7.17", + "@backstage/backend-common": "^0.1.1-alpha.22", "@types/express": "^4.17.6", "apollo-server": "^2.16.1", "apollo-server-express": "^2.16.1", @@ -36,7 +36,7 @@ "yn": "^4.0.0" }, "devDependencies": { - "@backstage/cli": "^0.1.1-alpha.21", + "@backstage/cli": "^0.1.1-alpha.22", "@types/supertest": "^2.0.8", "eslint-plugin-graphql": "^4.0.0", "msw": "^0.20.5", diff --git a/plugins/identity-backend/package.json b/plugins/identity-backend/package.json index 18db7c9220..a34049fb91 100644 --- a/plugins/identity-backend/package.json +++ b/plugins/identity-backend/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-identity-backend", - "version": "0.1.1-alpha.21", + "version": "0.1.1-alpha.22", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -20,7 +20,7 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@backstage/backend-common": "^0.1.1-alpha.21", + "@backstage/backend-common": "^0.1.1-alpha.22", "@types/express": "^4.17.6", "compression": "^1.7.4", "cors": "^2.8.5", @@ -33,7 +33,7 @@ "yn": "^4.0.0" }, "devDependencies": { - "@backstage/cli": "^0.1.1-alpha.21", + "@backstage/cli": "^0.1.1-alpha.22", "jest-fetch-mock": "^3.0.3" }, "files": [ diff --git a/plugins/jenkins/package.json b/plugins/jenkins/package.json index c62df08db2..398161f270 100644 --- a/plugins/jenkins/package.json +++ b/plugins/jenkins/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-jenkins", - "version": "0.1.1-alpha.21", + "version": "0.1.1-alpha.22", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -21,10 +21,10 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@backstage/catalog-model": "^0.1.1-alpha.21", - "@backstage/core": "^0.1.1-alpha.21", - "@backstage/plugin-catalog": "^0.1.1-alpha.21", - "@backstage/theme": "^0.1.1-alpha.21", + "@backstage/catalog-model": "^0.1.1-alpha.22", + "@backstage/core": "^0.1.1-alpha.22", + "@backstage/plugin-catalog": "^0.1.1-alpha.22", + "@backstage/theme": "^0.1.1-alpha.22", "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", "@material-ui/lab": "4.0.0-alpha.45", @@ -36,8 +36,8 @@ "react-use": "^15.3.3" }, "devDependencies": { - "@backstage/cli": "^0.1.1-alpha.21", - "@backstage/dev-utils": "^0.1.1-alpha.21", + "@backstage/cli": "^0.1.1-alpha.22", + "@backstage/dev-utils": "^0.1.1-alpha.22", "@testing-library/jest-dom": "^5.10.1", "@testing-library/react": "^10.4.1", "@testing-library/user-event": "^12.0.7", diff --git a/plugins/lighthouse/package.json b/plugins/lighthouse/package.json index 24a106e53d..c5522ae67a 100644 --- a/plugins/lighthouse/package.json +++ b/plugins/lighthouse/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-lighthouse", - "version": "0.1.1-alpha.21", + "version": "0.1.1-alpha.22", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -21,9 +21,9 @@ "start": "backstage-cli plugin:serve" }, "dependencies": { - "@backstage/config": "^0.1.1-alpha.21", - "@backstage/core": "^0.1.1-alpha.21", - "@backstage/theme": "^0.1.1-alpha.21", + "@backstage/config": "^0.1.1-alpha.22", + "@backstage/core": "^0.1.1-alpha.22", + "@backstage/theme": "^0.1.1-alpha.22", "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", "@material-ui/lab": "4.0.0-alpha.45", @@ -34,9 +34,9 @@ "react-use": "^15.3.3" }, "devDependencies": { - "@backstage/cli": "^0.1.1-alpha.21", - "@backstage/dev-utils": "^0.1.1-alpha.21", - "@backstage/test-utils": "^0.1.1-alpha.21", + "@backstage/cli": "^0.1.1-alpha.22", + "@backstage/dev-utils": "^0.1.1-alpha.22", + "@backstage/test-utils": "^0.1.1-alpha.22", "@testing-library/jest-dom": "^5.10.1", "@testing-library/react": "^10.4.1", "@testing-library/user-event": "^12.0.7", diff --git a/plugins/newrelic/package.json b/plugins/newrelic/package.json index 38a47bc0fc..0cadf440f7 100644 --- a/plugins/newrelic/package.json +++ b/plugins/newrelic/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-newrelic", - "version": "0.1.1-alpha.21", + "version": "0.1.1-alpha.22", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -21,8 +21,8 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@backstage/core": "^0.1.1-alpha.21", - "@backstage/theme": "^0.1.1-alpha.21", + "@backstage/core": "^0.1.1-alpha.22", + "@backstage/theme": "^0.1.1-alpha.22", "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", "@material-ui/lab": "4.0.0-alpha.45", @@ -31,8 +31,8 @@ "react-use": "^15.3.3" }, "devDependencies": { - "@backstage/cli": "^0.1.1-alpha.21", - "@backstage/dev-utils": "^0.1.1-alpha.21", + "@backstage/cli": "^0.1.1-alpha.22", + "@backstage/dev-utils": "^0.1.1-alpha.22", "@testing-library/jest-dom": "^5.10.1", "@testing-library/react": "^10.4.1", "@testing-library/user-event": "^12.0.7", diff --git a/plugins/proxy-backend/package.json b/plugins/proxy-backend/package.json index efda451fd5..fb1b82f3ea 100644 --- a/plugins/proxy-backend/package.json +++ b/plugins/proxy-backend/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-proxy-backend", - "version": "0.1.1-alpha.21", + "version": "0.1.1-alpha.22", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -19,8 +19,8 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@backstage/backend-common": "^0.1.1-alpha.21", - "@backstage/config": "^0.1.1-alpha.21", + "@backstage/backend-common": "^0.1.1-alpha.22", + "@backstage/config": "^0.1.1-alpha.22", "@types/express": "^4.17.6", "@types/http-proxy-middleware": "^0.19.3", "express": "^4.17.1", @@ -35,7 +35,7 @@ "yup": "^0.29.1" }, "devDependencies": { - "@backstage/cli": "^0.1.1-alpha.21", + "@backstage/cli": "^0.1.1-alpha.22", "@types/node-fetch": "^2.5.7", "@types/supertest": "^2.0.8", "@types/uuid": "^8.0.0", diff --git a/plugins/register-component/package.json b/plugins/register-component/package.json index 3dda51c3d5..8ccaae9914 100644 --- a/plugins/register-component/package.json +++ b/plugins/register-component/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-register-component", - "version": "0.1.1-alpha.21", + "version": "0.1.1-alpha.22", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -21,10 +21,10 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@backstage/catalog-model": "^0.1.1-alpha.21", - "@backstage/core": "^0.1.1-alpha.21", - "@backstage/plugin-catalog": "^0.1.1-alpha.21", - "@backstage/theme": "^0.1.1-alpha.21", + "@backstage/catalog-model": "^0.1.1-alpha.22", + "@backstage/core": "^0.1.1-alpha.22", + "@backstage/plugin-catalog": "^0.1.1-alpha.22", + "@backstage/theme": "^0.1.1-alpha.22", "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", "@material-ui/lab": "4.0.0-alpha.45", @@ -36,8 +36,8 @@ "react-use": "^15.3.3" }, "devDependencies": { - "@backstage/cli": "^0.1.1-alpha.21", - "@backstage/dev-utils": "^0.1.1-alpha.21", + "@backstage/cli": "^0.1.1-alpha.22", + "@backstage/dev-utils": "^0.1.1-alpha.22", "@testing-library/jest-dom": "^5.10.1", "@testing-library/react": "^10.4.1", "@testing-library/user-event": "^12.0.7", diff --git a/plugins/rollbar-backend/package.json b/plugins/rollbar-backend/package.json index 96a968ee46..969d3c79b9 100644 --- a/plugins/rollbar-backend/package.json +++ b/plugins/rollbar-backend/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-rollbar-backend", - "version": "0.1.1-alpha.21", + "version": "0.1.1-alpha.22", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -20,8 +20,8 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@backstage/backend-common": "^0.1.1-alpha.21", - "@backstage/config": "^0.1.1-alpha.21", + "@backstage/backend-common": "^0.1.1-alpha.22", + "@backstage/config": "^0.1.1-alpha.22", "@types/express": "^4.17.6", "axios": "^0.20.0", "camelcase-keys": "^6.2.2", @@ -37,7 +37,7 @@ "yn": "^4.0.0" }, "devDependencies": { - "@backstage/cli": "^0.1.1-alpha.21", + "@backstage/cli": "^0.1.1-alpha.22", "@types/supertest": "^2.0.8", "jest-fetch-mock": "^3.0.3", "supertest": "^4.0.2" diff --git a/plugins/rollbar/package.json b/plugins/rollbar/package.json index ae9d0c5bc5..cb37c6187a 100644 --- a/plugins/rollbar/package.json +++ b/plugins/rollbar/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-rollbar", - "version": "0.1.1-alpha.21", + "version": "0.1.1-alpha.22", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -21,10 +21,10 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@backstage/catalog-model": "^0.1.1-alpha.21", - "@backstage/core": "^0.1.1-alpha.21", - "@backstage/plugin-catalog": "^0.1.1-alpha.21", - "@backstage/theme": "^0.1.1-alpha.21", + "@backstage/catalog-model": "^0.1.1-alpha.22", + "@backstage/core": "^0.1.1-alpha.22", + "@backstage/plugin-catalog": "^0.1.1-alpha.22", + "@backstage/theme": "^0.1.1-alpha.22", "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", "@material-ui/lab": "4.0.0-alpha.45", @@ -37,9 +37,9 @@ "react-use": "^15.3.3" }, "devDependencies": { - "@backstage/cli": "^0.1.1-alpha.21", - "@backstage/dev-utils": "^0.1.1-alpha.21", - "@backstage/test-utils": "^0.1.1-alpha.21", + "@backstage/cli": "^0.1.1-alpha.22", + "@backstage/dev-utils": "^0.1.1-alpha.22", + "@backstage/test-utils": "^0.1.1-alpha.22", "@testing-library/jest-dom": "^5.10.1", "@testing-library/react": "^10.4.1", "@testing-library/react-hooks": "^3.3.0", diff --git a/plugins/scaffolder-backend/package.json b/plugins/scaffolder-backend/package.json index e0a9ae5129..0eaa33e1f8 100644 --- a/plugins/scaffolder-backend/package.json +++ b/plugins/scaffolder-backend/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-scaffolder-backend", - "version": "0.1.1-alpha.21", + "version": "0.1.1-alpha.22", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -20,9 +20,9 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@backstage/backend-common": "^0.1.1-alpha.21", - "@backstage/catalog-model": "^0.1.1-alpha.21", - "@backstage/config": "^0.1.1-alpha.21", + "@backstage/backend-common": "^0.1.1-alpha.22", + "@backstage/catalog-model": "^0.1.1-alpha.22", + "@backstage/config": "^0.1.1-alpha.22", "@gitbeaker/core": "^23.5.0", "@gitbeaker/node": "^23.5.0", "@octokit/rest": "^18.0.0", @@ -46,7 +46,7 @@ "yaml": "^1.10.0" }, "devDependencies": { - "@backstage/cli": "^0.1.1-alpha.21", + "@backstage/cli": "^0.1.1-alpha.22", "@octokit/types": "^5.4.1", "@types/fs-extra": "^9.0.1", "@types/git-url-parse": "^9.0.0", diff --git a/plugins/scaffolder/package.json b/plugins/scaffolder/package.json index 2efc2c270a..ab95e83201 100644 --- a/plugins/scaffolder/package.json +++ b/plugins/scaffolder/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-scaffolder", - "version": "0.1.1-alpha.21", + "version": "0.1.1-alpha.22", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -21,10 +21,10 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@backstage/catalog-model": "^0.1.1-alpha.21", - "@backstage/core": "^0.1.1-alpha.21", - "@backstage/plugin-catalog": "^0.1.1-alpha.21", - "@backstage/theme": "^0.1.1-alpha.21", + "@backstage/catalog-model": "^0.1.1-alpha.22", + "@backstage/core": "^0.1.1-alpha.22", + "@backstage/plugin-catalog": "^0.1.1-alpha.22", + "@backstage/theme": "^0.1.1-alpha.22", "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", "@material-ui/lab": "4.0.0-alpha.45", @@ -41,9 +41,9 @@ "swr": "^0.3.0" }, "devDependencies": { - "@backstage/cli": "^0.1.1-alpha.21", - "@backstage/dev-utils": "^0.1.1-alpha.21", - "@backstage/test-utils": "^0.1.1-alpha.21", + "@backstage/cli": "^0.1.1-alpha.22", + "@backstage/dev-utils": "^0.1.1-alpha.22", + "@backstage/test-utils": "^0.1.1-alpha.22", "@testing-library/jest-dom": "^5.10.1", "@testing-library/react": "^10.4.1", "@testing-library/user-event": "^12.0.7", diff --git a/plugins/sentry-backend/package.json b/plugins/sentry-backend/package.json index ffdcc18f7e..dbc7b48a1b 100644 --- a/plugins/sentry-backend/package.json +++ b/plugins/sentry-backend/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-sentry-backend", - "version": "0.1.1-alpha.21", + "version": "0.1.1-alpha.22", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -20,7 +20,7 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@backstage/backend-common": "^0.1.1-alpha.21", + "@backstage/backend-common": "^0.1.1-alpha.22", "@types/express": "^4.17.6", "axios": "^0.20.0", "compression": "^1.7.4", @@ -34,7 +34,7 @@ "yn": "^4.0.0" }, "devDependencies": { - "@backstage/cli": "^0.1.1-alpha.21", + "@backstage/cli": "^0.1.1-alpha.22", "jest-fetch-mock": "^3.0.3" }, "files": [ diff --git a/plugins/sentry/package.json b/plugins/sentry/package.json index e3b900ca8d..6e7daea15c 100644 --- a/plugins/sentry/package.json +++ b/plugins/sentry/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-sentry", - "version": "0.1.1-alpha.21", + "version": "0.1.1-alpha.22", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -21,9 +21,9 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@backstage/catalog-model": "^0.1.1-alpha.21", - "@backstage/core": "^0.1.1-alpha.21", - "@backstage/theme": "^0.1.1-alpha.21", + "@backstage/catalog-model": "^0.1.1-alpha.22", + "@backstage/core": "^0.1.1-alpha.22", + "@backstage/theme": "^0.1.1-alpha.22", "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", "@material-ui/lab": "4.0.0-alpha.45", @@ -36,8 +36,8 @@ "timeago.js": "^4.0.2" }, "devDependencies": { - "@backstage/cli": "^0.1.1-alpha.21", - "@backstage/dev-utils": "^0.1.1-alpha.21", + "@backstage/cli": "^0.1.1-alpha.22", + "@backstage/dev-utils": "^0.1.1-alpha.22", "@testing-library/jest-dom": "^5.10.1", "@testing-library/react": "^10.4.1", "@testing-library/user-event": "^12.0.7", diff --git a/plugins/tech-radar/package.json b/plugins/tech-radar/package.json index fc29d9c836..f25ec752e1 100644 --- a/plugins/tech-radar/package.json +++ b/plugins/tech-radar/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-tech-radar", - "version": "0.1.1-alpha.21", + "version": "0.1.1-alpha.22", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -21,9 +21,9 @@ "start": "backstage-cli plugin:serve" }, "dependencies": { - "@backstage/core": "^0.1.1-alpha.21", - "@backstage/test-utils": "^0.1.1-alpha.21", - "@backstage/theme": "^0.1.1-alpha.21", + "@backstage/core": "^0.1.1-alpha.22", + "@backstage/test-utils": "^0.1.1-alpha.22", + "@backstage/theme": "^0.1.1-alpha.22", "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", "@material-ui/lab": "4.0.0-alpha.45", @@ -35,8 +35,8 @@ "react-use": "^15.3.3" }, "devDependencies": { - "@backstage/cli": "^0.1.1-alpha.21", - "@backstage/dev-utils": "^0.1.1-alpha.21", + "@backstage/cli": "^0.1.1-alpha.22", + "@backstage/dev-utils": "^0.1.1-alpha.22", "@testing-library/jest-dom": "^5.10.1", "@testing-library/react": "^10.4.1", "@testing-library/user-event": "^12.0.7", diff --git a/plugins/techdocs-backend/package.json b/plugins/techdocs-backend/package.json index b9c2953d28..1ca8b3ee44 100644 --- a/plugins/techdocs-backend/package.json +++ b/plugins/techdocs-backend/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-techdocs-backend", - "version": "0.1.1-alpha.21", + "version": "0.1.1-alpha.22", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -20,9 +20,9 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@backstage/backend-common": "^0.1.1-alpha.21", - "@backstage/catalog-model": "^0.1.1-alpha.21", - "@backstage/config": "^0.1.1-alpha.21", + "@backstage/backend-common": "^0.1.1-alpha.22", + "@backstage/catalog-model": "^0.1.1-alpha.22", + "@backstage/config": "^0.1.1-alpha.22", "@types/dockerode": "^2.5.34", "@types/express": "^4.17.6", "command-exists-promise": "^2.0.2", @@ -38,7 +38,7 @@ "winston": "^3.2.1" }, "devDependencies": { - "@backstage/cli": "^0.1.1-alpha.21", + "@backstage/cli": "^0.1.1-alpha.22", "@types/node-fetch": "^2.5.7", "supertest": "^4.0.2" }, diff --git a/plugins/techdocs-backend/src/service/standaloneServer.ts b/plugins/techdocs-backend/src/service/standaloneServer.ts index 93233af4c0..7a39742b80 100644 --- a/plugins/techdocs-backend/src/service/standaloneServer.ts +++ b/plugins/techdocs-backend/src/service/standaloneServer.ts @@ -38,6 +38,7 @@ export async function startStandaloneServer( options: ServerOptions, ): Promise { const logger = options.logger.child({ service: 'techdocs-backend' }); + const config = ConfigReader.fromConfigs([]); logger.debug('Creating application...'); const preparers = new Preparers(); @@ -45,7 +46,7 @@ export async function startStandaloneServer( preparers.register('dir', directoryPreparer); const generators = new Generators(); - const techdocsGenerator = new TechdocsGenerator(logger); + const techdocsGenerator = new TechdocsGenerator(logger, config); generators.register('techdocs', techdocsGenerator); const publisher = new LocalPublish(logger); @@ -59,7 +60,7 @@ export async function startStandaloneServer( logger, publisher, dockerClient, - config: ConfigReader.fromConfigs([]), + config, }); const service = createServiceBuilder(module) .enableCors({ origin: 'http://localhost:3000' }) diff --git a/plugins/techdocs-backend/src/techdocs/stages/generate/generators.test.ts b/plugins/techdocs-backend/src/techdocs/stages/generate/generators.test.ts index b339aef9f8..a9303c3794 100644 --- a/plugins/techdocs-backend/src/techdocs/stages/generate/generators.test.ts +++ b/plugins/techdocs-backend/src/techdocs/stages/generate/generators.test.ts @@ -16,6 +16,7 @@ import { Generators, TechdocsGenerator } from './'; import { getVoidLogger } from '@backstage/backend-common'; +import { ConfigReader } from '@backstage/config'; const logger = getVoidLogger(); @@ -38,7 +39,10 @@ describe('generators', () => { it('should return correct registered generator', async () => { const generators = new Generators(); - const techdocs = new TechdocsGenerator(logger); + const techdocs = new TechdocsGenerator( + logger, + ConfigReader.fromConfigs([]), + ); generators.register('techdocs', techdocs); diff --git a/plugins/techdocs-backend/src/techdocs/stages/generate/techdocs.ts b/plugins/techdocs-backend/src/techdocs/stages/generate/techdocs.ts index cb89b42e38..ac73963f9a 100644 --- a/plugins/techdocs-backend/src/techdocs/stages/generate/techdocs.ts +++ b/plugins/techdocs-backend/src/techdocs/stages/generate/techdocs.ts @@ -18,6 +18,8 @@ import fs from 'fs-extra'; import path from 'path'; import os from 'os'; import { Logger } from 'winston'; +import { PassThrough } from 'stream'; +import { Config } from '@backstage/config'; import { GeneratorBase, @@ -26,18 +28,39 @@ import { } from './types'; import { runDockerContainer, runCommand } from './helpers'; -const commandExists = require('command-exists-promise'); +type TechdocsGeneratorOptions = { + // This option enables users to configure if they want to use TechDocs container + // or generate without the container. + // This is used to avoid running into Docker in Docker environment. + runGeneratorIn: string; +}; + +const createStream = (): [string[], PassThrough] => { + const log = [] as Array; + + const stream = new PassThrough(); + stream.on('data', chunk => { + const textValue = chunk.toString().trim(); + if (textValue?.length > 1) log.push(textValue); + }); + + return [log, stream]; +}; export class TechdocsGenerator implements GeneratorBase { private readonly logger: Logger; + private readonly options: TechdocsGeneratorOptions; - constructor(logger: Logger) { + constructor(logger: Logger, config: Config) { this.logger = logger; + this.options = { + runGeneratorIn: + config.getOptionalString('techdocs.generators.techdocs') ?? 'docker', + }; } public async run({ directory, - logStream, dockerClient, }: GeneratorRunOptions): Promise { const tmpdirPath = os.tmpdir(); @@ -46,35 +69,46 @@ export class TechdocsGenerator implements GeneratorBase { const resultDir = fs.mkdtempSync( path.join(tmpdirResolvedPath, 'techdocs-tmp-'), ); + const [log, logStream] = createStream(); try { - const mkdocsInstalled = await commandExists('mkdocs'); - if (mkdocsInstalled) { - await runCommand({ - command: 'mkdocs', - args: ['build', '-d', resultDir, '-v'], - options: { - cwd: directory, - }, - logStream, - }); - } else { - await runDockerContainer({ - imageName: 'spotify/techdocs', - args: ['build', '-d', '/result'], - logStream, - docsDir: directory, - resultDir, - dockerClient, - }); - this.logger.info( - `[TechDocs]: Successfully generated docs from ${directory} into ${resultDir}`, - ); + switch (this.options.runGeneratorIn) { + case 'local': + await runCommand({ + command: 'mkdocs', + args: ['build', '-d', resultDir, '-v'], + options: { + cwd: directory, + }, + logStream, + }); + this.logger.info( + `[TechDocs]: Successfully generated docs from ${directory} into ${resultDir} using local mkdocs`, + ); + break; + case 'docker': + await runDockerContainer({ + imageName: 'spotify/techdocs', + args: ['build', '-d', '/result'], + logStream, + docsDir: directory, + resultDir, + dockerClient, + }); + this.logger.info( + `[TechDocs]: Successfully generated docs from ${directory} into ${resultDir} using techdocs-container`, + ); + break; + default: + throw new Error( + `Invalid config value "${this.options.runGeneratorIn}" provided in 'techdocs.generators.techdocs'.`, + ); } } catch (error) { this.logger.debug( `[TechDocs]: Failed to generate docs from ${directory} into ${resultDir}`, ); + this.logger.debug(`[TechDocs]: Build failed with error: ${log}`); throw new Error( `Failed to generate docs from ${directory} into ${resultDir} with error ${error.message}`, ); diff --git a/plugins/techdocs/package.json b/plugins/techdocs/package.json index a3471c8142..d2f99cb824 100644 --- a/plugins/techdocs/package.json +++ b/plugins/techdocs/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-techdocs", - "version": "0.1.1-alpha.21", + "version": "0.1.1-alpha.22", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -22,12 +22,12 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@backstage/catalog-model": "^0.1.1-alpha.21", - "@backstage/core": "^0.1.1-alpha.21", - "@backstage/core-api": "^0.1.1-alpha.21", - "@backstage/plugin-catalog": "^0.1.1-alpha.21", - "@backstage/test-utils": "^0.1.1-alpha.21", - "@backstage/theme": "^0.1.1-alpha.21", + "@backstage/catalog-model": "^0.1.1-alpha.22", + "@backstage/core": "^0.1.1-alpha.22", + "@backstage/core-api": "^0.1.1-alpha.22", + "@backstage/plugin-catalog": "^0.1.1-alpha.22", + "@backstage/test-utils": "^0.1.1-alpha.22", + "@backstage/theme": "^0.1.1-alpha.22", "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", "@material-ui/lab": "4.0.0-alpha.45", @@ -40,8 +40,8 @@ "sanitize-html": "^1.27.0" }, "devDependencies": { - "@backstage/cli": "^0.1.1-alpha.21", - "@backstage/dev-utils": "^0.1.1-alpha.21", + "@backstage/cli": "^0.1.1-alpha.22", + "@backstage/dev-utils": "^0.1.1-alpha.22", "@testing-library/jest-dom": "^5.10.1", "@testing-library/react": "^10.4.1", "@testing-library/user-event": "^12.0.7", diff --git a/plugins/welcome/package.json b/plugins/welcome/package.json index a0dae6859d..741a680cd9 100644 --- a/plugins/welcome/package.json +++ b/plugins/welcome/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-welcome", - "version": "0.1.1-alpha.21", + "version": "0.1.1-alpha.22", "main": "src/index.ts", "types": "src/index.ts", "private": false, @@ -21,8 +21,8 @@ "start": "backstage-cli plugin:serve" }, "dependencies": { - "@backstage/core": "^0.1.1-alpha.21", - "@backstage/theme": "^0.1.1-alpha.21", + "@backstage/core": "^0.1.1-alpha.22", + "@backstage/theme": "^0.1.1-alpha.22", "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", "@material-ui/lab": "4.0.0-alpha.45", @@ -32,8 +32,8 @@ "react-use": "^15.3.3" }, "devDependencies": { - "@backstage/cli": "^0.1.1-alpha.21", - "@backstage/dev-utils": "^0.1.1-alpha.21", + "@backstage/cli": "^0.1.1-alpha.22", + "@backstage/dev-utils": "^0.1.1-alpha.22", "@testing-library/jest-dom": "^5.10.1", "@testing-library/react": "^10.4.1", "@testing-library/user-event": "^12.0.7",