diff --git a/.changeset/fluffy-lies-complain.md b/.changeset/fluffy-lies-complain.md new file mode 100644 index 0000000000..b770928e1c --- /dev/null +++ b/.changeset/fluffy-lies-complain.md @@ -0,0 +1,47 @@ +--- +'@backstage/techdocs-common': minor +--- + +Move the sanity checks of the publisher configurations to a dedicated `PublisherBase#getReadiness()` method instead of throwing an error when doing `Publisher.fromConfig(...)`. +You should include the check when your backend to get early feedback about a potential misconfiguration: + +```diff + // packages/backend/src/plugins/techdocs.ts + + export default async function createPlugin({ + logger, + config, + discovery, + reader, + }: PluginEnvironment): Promise { + // ... + + const publisher = await Publisher.fromConfig(config, { + logger, + discovery, + }) + ++ // checks if the publisher is working and logs the result ++ await publisher.getReadiness(); + + // Docker client (conditionally) used by the generators, based on techdocs.generators config. + const dockerClient = new Docker(); + + // ... +} +``` + +If you want to crash your application on invalid configurations, you can throw an `Error` to preserve the old behavior. +Please be aware that this is not the recommended for the use in a Backstage backend but might be helpful in CLI tools such as the `techdocs-cli`. + +```ts +const publisher = await Publisher.fromConfig(config, { + logger, + discovery, +}); + +const ready = await publisher.getReadiness(); +if (!ready.isAvailable) { + throw new Error('Invalid TechDocs publisher configuration'); +} +``` diff --git a/.changeset/metal-foxes-speak.md b/.changeset/metal-foxes-speak.md new file mode 100644 index 0000000000..45b8ae3c7c --- /dev/null +++ b/.changeset/metal-foxes-speak.md @@ -0,0 +1,5 @@ +--- +'@backstage/catalog-model': patch +--- + +Add getEntitySourceLocation helper diff --git a/.changeset/old-clouds-whisper.md b/.changeset/old-clouds-whisper.md new file mode 100644 index 0000000000..edaba13ad7 --- /dev/null +++ b/.changeset/old-clouds-whisper.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-kubernetes-backend': patch +--- + +Use `string` TypeScript type instead of `String`. diff --git a/.changeset/silent-papayas-dress.md b/.changeset/silent-papayas-dress.md new file mode 100644 index 0000000000..c3c05971fe --- /dev/null +++ b/.changeset/silent-papayas-dress.md @@ -0,0 +1,38 @@ +--- +'@backstage/plugin-scaffolder-backend': patch +--- + +Enable the JSON parsing of the response from templated variables in the `v2beta1` syntax. Previously if template parameters json strings they were left as strings, they are now parsed as JSON objects. + +Before: + +```yaml +- id: test + name: test-action + action: custom:run + input: + input: '{"hello":"ben"}' +``` + +Now: + +```yaml +- id: test + name: test-action + action: custom:run + input: + input: + hello: ben +``` + +Also added the `parseRepoUrl` and `json` helpers to the parameters syntax. You can now use these helpers to parse work with some `json` or `repoUrl` strings in templates. + +```yaml +- id: test + name: test-action + action: cookiecutter:fetch + input: + destination: '{{ parseRepoUrl parameters.repoUrl }}' +``` + +Will produce a parsed version of the `repoUrl` of type `{ repo: string, owner: string, host: string }` that you can use in your actions. Specifically `cookiecutter` with `{{ cookiecutter.destination.owner }}` like the `plugins/scaffolder-backend/sample-templates/v1beta2-demo/template.yaml` example. diff --git a/.changeset/techdocs-chatty-impalas-compare.md b/.changeset/techdocs-chatty-impalas-compare.md new file mode 100644 index 0000000000..26e7fd571d --- /dev/null +++ b/.changeset/techdocs-chatty-impalas-compare.md @@ -0,0 +1,6 @@ +--- +'@backstage/plugin-techdocs': patch +--- + +Reworked the TechDocs plugin to support using the configured company name instead of +'Backstage' in the page title. diff --git a/.changeset/tiny-games-hide.md b/.changeset/tiny-games-hide.md new file mode 100644 index 0000000000..2687174060 --- /dev/null +++ b/.changeset/tiny-games-hide.md @@ -0,0 +1,5 @@ +--- +'@backstage/catalog-client': patch +--- + +Make sure the `CatalogClient` escapes URL parameters correctly. diff --git a/.changeset/yellow-snails-hang.md b/.changeset/yellow-snails-hang.md new file mode 100644 index 0000000000..e908a98aee --- /dev/null +++ b/.changeset/yellow-snails-hang.md @@ -0,0 +1,33 @@ +--- +'@backstage/create-app': patch +--- + +Due to a change in the techdocs publishers, they don't check if they are able to reach e.g. the configured S3 bucket anymore. +This can be added again by the following change. Note that the backend process will no longer exit when it is not reachable but will only emit an error log message. +You should include the check when your backend to get early feedback about a potential misconfiguration: + +```diff + // packages/backend/src/plugins/techdocs.ts + + export default async function createPlugin({ + logger, + config, + discovery, + reader, + }: PluginEnvironment): Promise { + // ... + + const publisher = await Publisher.fromConfig(config, { + logger, + discovery, + }) + ++ // checks if the publisher is working and logs the result ++ await publisher.getReadiness(); + + // Docker client (conditionally) used by the generators, based on techdocs.generators config. + const dockerClient = new Docker(); + + // ... +} +``` diff --git a/.changeset/young-kids-remember.md b/.changeset/young-kids-remember.md new file mode 100644 index 0000000000..c07097234f --- /dev/null +++ b/.changeset/young-kids-remember.md @@ -0,0 +1,5 @@ +--- +'@backstage/cli': patch +--- + +Disable hot reloading in CI environments. diff --git a/.tugboat/tugboat.app-config.production.yaml b/.tugboat/tugboat.app-config.production.yaml index f606574d89..b264d53f3f 100644 --- a/.tugboat/tugboat.app-config.production.yaml +++ b/.tugboat/tugboat.app-config.production.yaml @@ -1,13 +1,10 @@ app: title: Backstage Tugboat Preview - baseUrl: - $env: TUGBOAT_DEFAULT_SERVICE_URL + baseUrl: ${TUGBOAT_DEFAULT_SERVICE_URL} backend: - baseUrl: - $env: TUGBOAT_DEFAULT_SERVICE_URL + baseUrl: ${TUGBOAT_DEFAULT_SERVICE_URL} cors: - origin: - $env: TUGBOAT_DEFAULT_SERVICE_URL + origin: ${TUGBOAT_DEFAULT_SERVICE_URL} methods: [GET, POST, PUT, DELETE] credentials: true diff --git a/app-config.yaml b/app-config.yaml index 309fa28234..afbda40852 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -40,47 +40,40 @@ proxy: '/circleci/api': target: https://circleci.com/api/v1.1 headers: - Circle-Token: - $env: CIRCLECI_AUTH_TOKEN + Circle-Token: ${CIRCLECI_AUTH_TOKEN} '/jenkins/api': target: http://localhost:8080 headers: - Authorization: - $env: JENKINS_BASIC_AUTH_HEADER + Authorization: ${JENKINS_BASIC_AUTH_HEADER} '/travisci/api': target: https://api.travis-ci.com changeOrigin: true headers: - Authorization: - $env: TRAVISCI_AUTH_TOKEN + Authorization: ${TRAVISCI_AUTH_TOKEN} travis-api-version: '3' '/newrelic/apm/api': target: https://api.newrelic.com/v2 headers: - X-Api-Key: - $env: NEW_RELIC_REST_API_KEY + X-Api-Key: ${NEW_RELIC_REST_API_KEY} '/pagerduty': target: https://api.pagerduty.com headers: - Authorization: - $env: PAGERDUTY_TOKEN + Authorization: ${PAGERDUTY_TOKEN} '/buildkite/api': target: https://api.buildkite.com/v2/ headers: - Authorization: - $env: BUILDKITE_TOKEN + Authorization: ${BUILDKITE_TOKEN} '/sentry/api': target: https://sentry.io/api/ allowedMethods: ['GET'] headers: - Authorization: - $env: SENTRY_TOKEN + Authorization: ${SENTRY_TOKEN} organization: name: My Company @@ -124,36 +117,28 @@ kafka: integrations: github: - host: github.com - token: - $env: GITHUB_TOKEN + token: ${GITHUB_TOKEN} ### Example for how to add your GitHub Enterprise instance using the API: # - host: ghe.example.net # apiBaseUrl: https://ghe.example.net/api/v3 - # token: - # $env: GHE_TOKEN + # token: ${GHE_TOKEN} ### Example for how to add your GitHub Enterprise instance using raw HTTP fetches (token is optional): # - host: ghe.example.net # rawBaseUrl: https://ghe.example.net/raw - # token: - # $env: GHE_TOKEN + # token: ${GHE_TOKEN} gitlab: - host: gitlab.com - token: - $env: GITLAB_TOKEN + token: ${GITLAB_TOKEN} bitbucket: - host: bitbucket.org - username: - $env: BITBUCKET_USERNAME - appPassword: - $env: BITBUCKET_APP_PASSWORD + username: ${BITBUCKET_USERNAME} + appPassword: ${BITBUCKET_APP_PASSWORD} azure: - host: dev.azure.com - token: - $env: AZURE_TOKEN + token: ${AZURE_TOKEN} # googleGcs: # clientEmail: 'example@example.com' -# privateKey: -# $env: GCS_PRIVATE_KEY +# privateKey: ${GCS_PRIVATE_KEY} catalog: rules: @@ -172,21 +157,18 @@ catalog: githubOrg: providers: - target: https://github.com - token: - $env: GITHUB_TOKEN + token: ${GITHUB_TOKEN} #### Example for how to add your GitHub Enterprise instance using the API: # - target: https://ghe.example.net # apiBaseUrl: https://ghe.example.net/api - # token: - # $env: GHE_TOKEN + # token: ${GHE_TOKEN} ldapOrg: ### Example for how to add your enterprise LDAP server # providers: # - target: ldaps://ds.example.net # bind: # dn: uid=ldap-reader-user,ou=people,ou=example,dc=example,dc=net - # secret: - # $env: LDAP_SECRET + # secret: ${LDAP_SECRET} # users: # dn: ou=people,ou=example,dc=example,dc=net # options: @@ -202,12 +184,9 @@ catalog: #providers: # - target: https://graph.microsoft.com/v1.0 # authority: https://login.microsoftonline.com - # tenantId: - # $env: MICROSOFT_GRAPH_TENANT_ID - # clientId: - # $env: MICROSOFT_GRAPH_CLIENT_ID - # clientSecret: - # $env: MICROSOFT_GRAPH_CLIENT_SECRET_TOKEN + # tenantId: ${MICROSOFT_GRAPH_TENANT_ID} + # clientId: ${MICROSOFT_GRAPH_CLIENT_ID} + # clientSecret: ${MICROSOFT_GRAPH_CLIENT_SECRET_TOKEN} # userFilter: accountEnabled eq true and userType eq 'member' # groupFilter: securityEnabled eq false and mailEnabled eq true and groupTypes/any(c:c+eq+'Unified') @@ -255,27 +234,22 @@ catalog: scaffolder: github: - token: - $env: GITHUB_TOKEN + token: ${GITHUB_TOKEN} visibility: public # or 'internal' or 'private' gitlab: api: baseUrl: https://gitlab.com - token: - $env: GITLAB_TOKEN + token: ${GITLAB_TOKEN} visibility: public # or 'internal' or 'private' azure: baseUrl: https://dev.azure.com/{your-organization} api: - token: - $env: AZURE_TOKEN + token: ${AZURE_TOKEN} bitbucket: api: host: https://bitbucket.org - username: - $env: BITBUCKET_USERNAME - token: - $env: BITBUCKET_TOKEN + username: ${BITBUCKET_USERNAME} + token: ${BITBUCKET_TOKEN} visibility: public # or or 'private' auth: @@ -286,89 +260,59 @@ auth: providers: google: development: - clientId: - $env: AUTH_GOOGLE_CLIENT_ID - clientSecret: - $env: AUTH_GOOGLE_CLIENT_SECRET + clientId: ${AUTH_GOOGLE_CLIENT_ID} + clientSecret: ${AUTH_GOOGLE_CLIENT_SECRET} github: development: - clientId: - $env: AUTH_GITHUB_CLIENT_ID - clientSecret: - $env: AUTH_GITHUB_CLIENT_SECRET - enterpriseInstanceUrl: - $env: AUTH_GITHUB_ENTERPRISE_INSTANCE_URL + clientId: ${AUTH_GITHUB_CLIENT_ID} + clientSecret: ${AUTH_GITHUB_CLIENT_SECRET} + enterpriseInstanceUrl: ${AUTH_GITHUB_ENTERPRISE_INSTANCE_URL} gitlab: development: - clientId: - $env: AUTH_GITLAB_CLIENT_ID - clientSecret: - $env: AUTH_GITLAB_CLIENT_SECRET - audience: - $env: GITLAB_BASE_URL + clientId: ${AUTH_GITLAB_CLIENT_ID} + clientSecret: ${AUTH_GITLAB_CLIENT_SECRET} + audience: ${GITLAB_BASE_URL} saml: entryPoint: 'http://localhost:7001/' issuer: 'passport-saml' okta: development: - clientId: - $env: AUTH_OKTA_CLIENT_ID - clientSecret: - $env: AUTH_OKTA_CLIENT_SECRET - audience: - $env: AUTH_OKTA_AUDIENCE + clientId: ${AUTH_OKTA_CLIENT_ID} + clientSecret: ${AUTH_OKTA_CLIENT_SECRET} + audience: ${AUTH_OKTA_AUDIENCE} oauth2: development: - clientId: - $env: AUTH_OAUTH2_CLIENT_ID - clientSecret: - $env: AUTH_OAUTH2_CLIENT_SECRET - authorizationUrl: - $env: AUTH_OAUTH2_AUTH_URL - tokenUrl: - $env: AUTH_OAUTH2_TOKEN_URL + clientId: ${AUTH_OAUTH2_CLIENT_ID} + clientSecret: ${AUTH_OAUTH2_CLIENT_SECRET} + authorizationUrl: ${AUTH_OAUTH2_AUTH_URL} + tokenUrl: ${AUTH_OAUTH2_TOKEN_URL} ### # provide a list of scopes as needed for your OAuth2 Server: # # scope: saml-login-selector openid profile email oidc: development: - metadataUrl: - $env: AUTH_OIDC_METADATA_URL - clientId: - $env: AUTH_OIDC_CLIENT_ID - clientSecret: - $env: AUTH_OIDC_CLIENT_SECRET - authorizationUrl: - $env: AUTH_OIDC_AUTH_URL - tokenUrl: - $env: AUTH_OIDC_TOKEN_URL - tokenSignedResponseAlg: - $env: AUTH_OIDC_TOKEN_SIGNED_RESPONSE_ALG + metadataUrl: ${AUTH_OIDC_METADATA_URL} + clientId: ${AUTH_OIDC_CLIENT_ID} + clientSecret: ${AUTH_OIDC_CLIENT_SECRET} + authorizationUrl: ${AUTH_OIDC_AUTH_URL} + tokenUrl: ${AUTH_OIDC_TOKEN_URL} + tokenSignedResponseAlg: ${AUTH_OIDC_TOKEN_SIGNED_RESPONSE_ALG} auth0: development: - clientId: - $env: AUTH_AUTH0_CLIENT_ID - clientSecret: - $env: AUTH_AUTH0_CLIENT_SECRET - domain: - $env: AUTH_AUTH0_DOMAIN + clientId: ${AUTH_AUTH0_CLIENT_ID} + clientSecret: ${AUTH_AUTH0_CLIENT_SECRET} + domain: ${AUTH_AUTH0_DOMAIN} microsoft: development: - clientId: - $env: AUTH_MICROSOFT_CLIENT_ID - clientSecret: - $env: AUTH_MICROSOFT_CLIENT_SECRET - tenantId: - $env: AUTH_MICROSOFT_TENANT_ID + clientId: ${AUTH_MICROSOFT_CLIENT_ID} + clientSecret: ${AUTH_MICROSOFT_CLIENT_SECRET} + tenantId: ${AUTH_MICROSOFT_TENANT_ID} onelogin: development: - clientId: - $env: AUTH_ONELOGIN_CLIENT_ID - clientSecret: - $env: AUTH_ONELOGIN_CLIENT_SECRET - issuer: - $env: AUTH_ONELOGIN_ISSUER + clientId: ${AUTH_ONELOGIN_CLIENT_ID} + clientSecret: ${AUTH_ONELOGIN_CLIENT_SECRET} + issuer: ${AUTH_ONELOGIN_ISSUER} costInsights: engineerCost: 200000 products: diff --git a/contrib/chart/backstage/values.yaml b/contrib/chart/backstage/values.yaml index 7b26d2bccd..bd80bc22b6 100644 --- a/contrib/chart/backstage/values.yaml +++ b/contrib/chart/backstage/values.yaml @@ -127,68 +127,47 @@ appConfig: development: appOrigin: 'http://localhost:3000/' secure: false - clientId: - $env: AUTH_GOOGLE_CLIENT_ID - clientSecret: - $env: AUTH_GOOGLE_CLIENT_SECRET + clientId: ${AUTH_GOOGLE_CLIENT_ID} + clientSecret: ${AUTH_GOOGLE_CLIENT_SECRET} github: development: appOrigin: 'http://localhost:3000/' secure: false - clientId: - $env: AUTH_GITHUB_CLIENT_ID - clientSecret: - $env: AUTH_GITHUB_CLIENT_SECRET - enterpriseInstanceUrl: - $env: AUTH_GITHUB_ENTERPRISE_INSTANCE_URL + clientId: ${AUTH_GITHUB_CLIENT_ID} + clientSecret: ${AUTH_GITHUB_CLIENT_SECRET} + enterpriseInstanceUrl: ${AUTH_GITHUB_ENTERPRISE_INSTANCE_URL} gitlab: development: appOrigin: 'http://localhost:3000/' secure: false - clientId: - $env: AUTH_GITLAB_CLIENT_ID - clientSecret: - $env: AUTH_GITLAB_CLIENT_SECRET - audience: - $env: GITLAB_BASE_URL + clientId: ${AUTH_GITLAB_CLIENT_ID} + clientSecret: ${AUTH_GITLAB_CLIENT_SECRET} + audience: ${GITLAB_BASE_URL} okta: development: appOrigin: 'http://localhost:3000/' secure: false - clientId: - $env: AUTH_OKTA_CLIENT_ID - clientSecret: - $env: AUTH_OKTA_CLIENT_SECRET - audience: - $env: AUTH_OKTA_AUDIENCE + clientId: ${AUTH_OKTA_CLIENT_ID} + clientSecret: ${AUTH_OKTA_CLIENT_SECRET} + audience: ${AUTH_OKTA_AUDIENCE} oauth2: development: appOrigin: 'http://localhost:3000/' secure: false - clientId: - $env: AUTH_OAUTH2_CLIENT_ID - clientSecret: - $env: AUTH_OAUTH2_CLIENT_SECRET - authorizationURL: - $env: AUTH_OAUTH2_AUTH_URL - tokenURL: - $env: AUTH_OAUTH2_TOKEN_URL + clientId: ${AUTH_OAUTH2_CLIENT_ID} + clientSecret: ${AUTH_OAUTH2_CLIENT_SECRET} + authorizationURL: ${AUTH_OAUTH2_AUTH_URL} + tokenURL: ${AUTH_OAUTH2_TOKEN_URL} auth0: development: - clientId: - $env: AUTH_AUTH0_CLIENT_ID - clientSecret: - $env: AUTH_AUTH0_CLIENT_SECRET - domain: - $env: AUTH_AUTH0_DOMAIN + clientId: ${AUTH_AUTH0_CLIENT_ID} + clientSecret: ${AUTH_AUTH0_CLIENT_SECRET} + domain: ${AUTH_AUTH0_DOMAIN} microsoft: development: - clientId: - $env: AUTH_MICROSOFT_CLIENT_ID - clientSecret: - $env: AUTH_MICROSOFT_CLIENT_SECRET - tenantId: - $env: AUTH_MICROSOFT_TENANT_ID + clientId: ${AUTH_MICROSOFT_CLIENT_ID} + clientSecret: ${AUTH_MICROSOFT_CLIENT_SECRET} + tenantId: ${AUTH_MICROSOFT_TENANT_ID} auth: google: diff --git a/docs/auth/auth-backend-classes.md b/docs/auth/auth-backend-classes.md index a99ca119f0..5b5950f6c3 100644 --- a/docs/auth/auth-backend-classes.md +++ b/docs/auth/auth-backend-classes.md @@ -95,40 +95,27 @@ auth: providers: google: development: - clientId: - $env: AUTH_GOOGLE_CLIENT_ID - clientSecret: - $env: AUTH_GOOGLE_CLIENT_SECRET + clientId: ${AUTH_GOOGLE_CLIENT_ID} + clientSecret: ${AUTH_GOOGLE_CLIENT_SECRET} github: development: - clientId: - $env: AUTH_GITHUB_CLIENT_ID - clientSecret: - $env: AUTH_GITHUB_CLIENT_SECRET - enterpriseInstanceUrl: - $env: AUTH_GITHUB_ENTERPRISE_INSTANCE_URL + clientId: ${AUTH_GITHUB_CLIENT_ID} + clientSecret: ${AUTH_GITHUB_CLIENT_SECRET} + enterpriseInstanceUrl: ${AUTH_GITHUB_ENTERPRISE_INSTANCE_URL} gitlab: development: - clientId: - $env: + clientId: ${AUTH_GITLAB_CLIENT_ID} oauth2: development: - clientId: - $env: AUTH_OAUTH2_CLIENT_ID - clientSecret: - $env: AUTH_OAUTH2_CLIENT_SECRET - authorizationUrl: - $env: AUTH_OAUTH2_AUTH_URL - tokenUrl: - $env: AUTH_OAUTH2_TOKEN_URL - scope: - $env: AUTH_OAUTH2_SCOPE + clientId: ${AUTH_OAUTH2_CLIENT_ID} + clientSecret: ${AUTH_OAUTH2_CLIENT_SECRET} + authorizationUrl: ${AUTH_OAUTH2_AUTH_URL} + tokenUrl: ${AUTH_OAUTH2_TOKEN_URL} + scope: ${AUTH_OAUTH2_SCOPE} saml: - entryPoint: - $env: AUTH_SAML_ENTRY_POINT - issuer: - $env: AUTH_SAML_ISSUER - ... + entryPoint: ${AUTH_SAML_ENTRY_POINT} + issuer: ${AUTH_SAML_ISSUER} + ... ``` ## Implementing Your Own Auth Wrapper diff --git a/docs/conf/writing.md b/docs/conf/writing.md index ac8509db0a..c7b3bfe296 100644 --- a/docs/conf/writing.md +++ b/docs/conf/writing.md @@ -129,6 +129,9 @@ variable. $env: MY_SECRET ``` +Note however, that it's often more convenient to use +[environment variable substitution](#environment-variable-substitution) instead. + ### File Includes This reads a string value from the entire contents of a text file. The file path diff --git a/docs/features/kubernetes/configuration.md b/docs/features/kubernetes/configuration.md index 4ac324a430..b96b1461de 100644 --- a/docs/features/kubernetes/configuration.md +++ b/docs/features/kubernetes/configuration.md @@ -26,8 +26,7 @@ kubernetes: name: minikube authProvider: 'serviceAccount' skipTLSVerify: false - serviceAccountToken: - $env: K8S_MINIKUBE_TOKEN + serviceAccountToken: ${K8S_MINIKUBE_TOKEN} - url: http://127.0.0.2:9999 name: aws-cluster-1 authProvider: 'aws' diff --git a/docs/features/software-catalog/external-integrations.md b/docs/features/software-catalog/external-integrations.md index e278bc7a1a..247324a46c 100644 --- a/docs/features/software-catalog/external-integrations.md +++ b/docs/features/software-catalog/external-integrations.md @@ -156,8 +156,18 @@ The key points to note are: - Call `emit` any number of times with the results of that process - Finally return `true` -You should now be able to instantiate this class in your backend, and add it to -the `CatalogBuilder` using the `addProcessors` method. +You should now be able to add this class to your backend in +`packages/backend/src/plugins/catalog.ts`: + +```diff ++ import { SystemXReaderProcessor } from '../path/to/class'; + +export default async function createPlugin( + env: PluginEnvironment, +): Promise { + const builder = new CatalogBuilder(env); ++ builder.addProcessor(new SystemXReaderProcessor(env.reader)); +``` Start up the backend - it should now start reading from the previously registered location and you'll see your entities start to appear in Backstage. diff --git a/docs/features/software-catalog/well-known-annotations.md b/docs/features/software-catalog/well-known-annotations.md index 8ec4813799..0053c14b79 100644 --- a/docs/features/software-catalog/well-known-annotations.md +++ b/docs/features/software-catalog/well-known-annotations.md @@ -92,12 +92,14 @@ view and edit links need changing. # Example: metadata: annotations: - backstage.io/source-location: github:https://github.com/my-org/my-service + backstage.io/source-location: url:https://github.com/my-org/my-service/ ``` A `Location` reference that points to the source code of the entity (typically a `Component`). Useful when catalog files do not get ingested from the source code -repository itself. +repository itself. If the URL points to a folder, it is important that it is +suffixed with a `'/'` in order for relative path resolution to work +consistently. ### jenkins.io/github-folder diff --git a/docs/features/software-templates/installation.md b/docs/features/software-templates/installation.md index b3c9f878d7..105b7e5f96 100644 --- a/docs/features/software-templates/installation.md +++ b/docs/features/software-templates/installation.md @@ -189,8 +189,7 @@ public within the enterprise. integrations: github: - host: github.com - token: - $env: GITHUB_TOKEN + token: ${GITHUB_TOKEN} scaffolder: github: @@ -207,8 +206,7 @@ instance: integrations: gitlab: - host: gitlab.com - token: - $env: GITLAB_TOKEN + token: ${GITLAB_TOKEN} ``` #### Bitbucket @@ -221,8 +219,7 @@ following: integrations: bitbucket: - host: bitbucket.org - token: - $env: BITBUCKET_TOKEN + token: ${BITBUCKET_TOKEN} ``` or @@ -231,10 +228,8 @@ or integrations: bitbucket: - host: bitbucket.org - appPassword: - $env: BITBUCKET_APP_PASSWORD - username: - $env: BITBUCKET_USERNAME + appPassword: ${BITBUCKET_APP_PASSWORD} + username: ${BITBUCKET_USERNAME} ``` #### Azure DevOps @@ -249,8 +244,7 @@ verified. integrations: azure: - host: dev.azure.com - token: - $env: AZURE_TOKEN + token: ${AZURE_TOKEN} ``` ### Running the Backend diff --git a/docs/features/techdocs/configuration.md b/docs/features/techdocs/configuration.md index 9bf2c25926..c7876f3874 100644 --- a/docs/features/techdocs/configuration.md +++ b/docs/features/techdocs/configuration.md @@ -65,22 +65,18 @@ techdocs: # https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/loading-node-credentials-environment.html # https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/loading-node-credentials-shared.html credentials: - accessKeyId: - $env: TECHDOCS_AWSS3_ACCESS_KEY_ID_CREDENTIAL - secretAccessKey: - $env: TECHDOCS_AWSS3_SECRET_ACCESS_KEY_CREDENTIAL + accessKeyId: ${TECHDOCS_AWSS3_ACCESS_KEY_ID_CREDENTIAL} + secretAccessKey: ${TECHDOCS_AWSS3_SECRET_ACCESS_KEY_CREDENTIAL} # (Optional) AWS Region of the bucket. # If not set, AWS_REGION environment variable or aws config file will be used. # https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/setting-region.html - region: - $env: AWS_REGION + region: ${AWS_REGION} # (Optional) Endpoint URI to send requests to. # If not set, the default endpoint is built from the configured region. # https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#constructor-property - endpoint: - $env: AWS_ENDPOINT + endpoint: ${AWS_ENDPOINT} # Required when techdocs.publisher.type is set to 'azureBlobStorage'. Skip otherwise. @@ -91,13 +87,11 @@ techdocs: # (Required) An account name is required to write to a storage blob container. # https://docs.microsoft.com/en-us/rest/api/storageservices/authorize-with-shared-key credentials: - accountName: - $env: TECHDOCS_AZURE_BLOB_STORAGE_ACCOUNT_NAME + accountName: ${TECHDOCS_AZURE_BLOB_STORAGE_ACCOUNT_NAME} # (Optional) An account key is required to write to a storage container. # If missing,AZURE_TENANT_ID, AZURE_CLIENT_ID, AZURE_CLIENT_SECRET environment variable will be used. # https://docs.microsoft.com/en-us/azure/storage/common/storage-auth?toc=/azure/storage/blobs/toc.json - accountKey: - $env: TECHDOCS_AZURE_BLOB_STORAGE_ACCOUNT_KEY + accountKey: ${TECHDOCS_AZURE_BLOB_STORAGE_ACCOUNT_KEY} # (Optional and Legacy) TechDocs makes API calls to techdocs-backend using this URL. e.g. get docs of an entity, get metadata, etc. # You don't have to specify this anymore. diff --git a/docs/features/techdocs/using-cloud-storage.md b/docs/features/techdocs/using-cloud-storage.md index fd8f320859..d5a3a68425 100644 --- a/docs/features/techdocs/using-cloud-storage.md +++ b/docs/features/techdocs/using-cloud-storage.md @@ -95,8 +95,7 @@ techdocs: type: 'googleGcs' googleGcs: bucketName: 'name-of-techdocs-storage-bucket' - credentials: - $env: GOOGLE_APPLICATION_CREDENTIALS + credentials: ${GOOGLE_APPLICATION_CREDENTIALS} ``` **4. That's it!** @@ -179,13 +178,10 @@ techdocs: type: 'awsS3' awsS3: bucketName: 'name-of-techdocs-storage-bucket' - region: - $env: AWS_REGION + region: ${AWS_REGION} credentials: - accessKeyId: - $env: AWS_ACCESS_KEY_ID - secretAccessKey: - $env: AWS_SECRET_ACCESS_KEY + accessKeyId: ${AWS_ACCESS_KEY_ID} + secretAccessKey: ${AWS_SECRET_ACCESS_KEY} ``` Refer to the @@ -202,8 +198,7 @@ techdocs: type: 'awsS3' awsS3: bucketName: 'name-of-techdocs-storage-bucket' - region: - $env: AWS_REGION + region: ${AWS_REGION} credentials: roleArn: arn:aws:iam::123456789012:role/my-backstage-role ``` @@ -276,8 +271,7 @@ techdocs: azureBlobStorage: containerName: 'name-of-techdocs-storage-bucket' credentials: - accountName: - $env: TECHDOCS_AZURE_BLOB_STORAGE_ACCOUNT_NAME + accountName: ${TECHDOCS_AZURE_BLOB_STORAGE_ACCOUNT_NAME} ``` **3b. Authentication using app-config.yaml** @@ -297,10 +291,8 @@ techdocs: azureBlobStorage: containerName: 'name-of-techdocs-storage-bucket' credentials: - accountName: - $env: TECHDOCS_AZURE_BLOB_STORAGE_ACCOUNT_NAME - accountKey: - $env: TECHDOCS_AZURE_BLOB_STORAGE_ACCOUNT_KEY + accountName: ${TECHDOCS_AZURE_BLOB_STORAGE_ACCOUNT_NAME} + accountKey: ${TECHDOCS_AZURE_BLOB_STORAGE_ACCOUNT_KEY} ``` **4. That's it!** @@ -361,20 +353,13 @@ techdocs: openStackSwift: containerName: 'name-of-techdocs-storage-bucket' credentials: - userName: - $env: OPENSTACK_SWIFT_STORAGE_USERNAME - password: - $env: OPENSTACK_SWIFT_STORAGE_PASSWORD - authUrl: - $env: OPENSTACK_SWIFT_STORAGE_AUTH_URL - keystoneAuthVersion: - $env: OPENSTACK_SWIFT_STORAGE_AUTH_VERSION - domainId: - $env: OPENSTACK_SWIFT_STORAGE_DOMAIN_ID - domainName: - $env: OPENSTACK_SWIFT_STORAGE_DOMAIN_NAME - region: - $env: OPENSTACK_SWIFT_STORAGE_REGION + userName: ${OPENSTACK_SWIFT_STORAGE_USERNAME} + password: ${OPENSTACK_SWIFT_STORAGE_PASSWORD} + authUrl: ${OPENSTACK_SWIFT_STORAGE_AUTH_URL} + keystoneAuthVersion: ${OPENSTACK_SWIFT_STORAGE_AUTH_VERSION} + domainId: ${OPENSTACK_SWIFT_STORAGE_DOMAIN_ID} + domainName: ${OPENSTACK_SWIFT_STORAGE_DOMAIN_NAME} + region: ${OPENSTACK_SWIFT_STORAGE_REGION} ``` **4. That's it!** diff --git a/docs/getting-started/configure-app-with-plugins.md b/docs/getting-started/configure-app-with-plugins.md index e5b4d08a3f..cdef932127 100644 --- a/docs/getting-started/configure-app-with-plugins.md +++ b/docs/getting-started/configure-app-with-plugins.md @@ -60,8 +60,7 @@ proxy: '/circleci/api': target: https://circleci.com/api/v1.1 headers: - Circle-Token: - $env: CIRCLECI_AUTH_TOKEN + Circle-Token: ${CIRCLECI_AUTH_TOKEN} ``` ### Adding a plugin page to the Sidebar diff --git a/docs/integrations/google-cloud-storage/locations.md b/docs/integrations/google-cloud-storage/locations.md index e3f6b72847..bcdf612b9a 100644 --- a/docs/integrations/google-cloud-storage/locations.md +++ b/docs/integrations/google-cloud-storage/locations.md @@ -24,10 +24,8 @@ Explicit credentials can be set in the following format: ```yaml integrations: googleGcs: - clientEmail: - $env: GCS_CLIENT_EMAIL - privateKey: - $env: GCS_PRIVATE_KEY + clientEmail: ${GCS_CLIENT_EMAIL} + privateKey: ${GCS_PRIVATE_KEY} ``` Then make sure the environment variables `GCS_CLIENT_EMAIL` and diff --git a/docs/plugins/proxying.md b/docs/plugins/proxying.md index f2d7d3c12a..50c56af7a6 100644 --- a/docs/plugins/proxying.md +++ b/docs/plugins/proxying.md @@ -40,8 +40,9 @@ proxy: '/larger-example/v1': target: http://larger.example.com:8080/svc.v1 headers: - Authorization: - $env: EXAMPLE_AUTH_HEADER + Authorization: ${EXAMPLE_AUTH_HEADER} + # ...or interpolating a value into part of a string, + # Authorization: Bearer ${EXAMPLE_AUTH_TOKEN} ``` Each key under the proxy configuration entry is a route to match, below the diff --git a/docs/tutorials/quickstart-app-auth.md b/docs/tutorials/quickstart-app-auth.md index d322e2ec3c..510b697221 100644 --- a/docs/tutorials/quickstart-app-auth.md +++ b/docs/tutorials/quickstart-app-auth.md @@ -79,13 +79,10 @@ auth: providers: github: development: - clientId: - $env: AUTH_GITHUB_CLIENT_ID - clientSecret: - $env: AUTH_GITHUB_CLIENT_SECRET - ## uncomment the following two lines if using enterprise - # enterpriseInstanceUrl: - # $env: AUTH_GITHUB_ENTERPRISE_INSTANCE_URL + clientId: ${AUTH_GITHUB_CLIENT_ID} + clientSecret: ${AUTH_GITHUB_CLIENT_SECRET} + ## uncomment the following line if using enterprise + # enterpriseInstanceUrl: ${AUTH_GITHUB_ENTERPRISE_INSTANCE_URL} ``` ### 2. Generate a GitHub client ID and secret @@ -122,10 +119,8 @@ auth: providers: gitlab: development: - clientId: - $env: AUTH_GITLAB_CLIENT_ID - clientSecret: - $env: AUTH_GITLAB_CLIENT_SECRET + clientId: ${AUTH_GITLAB_CLIENT_ID} + clientSecret: ${AUTH_GITLAB_CLIENT_SECRET} audience: https://gitlab.com # Or your self-hosted GitLab instance URL ``` @@ -172,10 +167,8 @@ auth: providers: google: development: - clientId: - $env: AUTH_GOOGLE_CLIENT_ID - clientSecret: - $env: AUTH_GOOGLE_CLIENT_SECRET + clientId: ${AUTH_GOOGLE_CLIENT_ID} + clientSecret: ${AUTH_GOOGLE_CLIENT_SECRET} ``` ### 2. Generate Google Credentials in Google Cloud console @@ -216,12 +209,9 @@ auth: providers: microsoft: development: - clientId: - $env: AUTH_MICROSOFT_CLIENT_ID - clientSecret: - $env: AUTH_MICROSOFT_CLIENT_SECRET - tenantId: - $env: AUTH_MICROSOFT_TENANT_ID + clientId: ${AUTH_MICROSOFT_CLIENT_ID} + clientSecret: ${AUTH_MICROSOFT_CLIENT_SECRET} + tenantId: ${AUTH_MICROSOFT_TENANT_ID} ``` ### 2. Create a Microsoft App Registration in Microsoft Portal @@ -264,12 +254,9 @@ auth: providers: auth0: development: - clientId: - $env: AUTH_AUTH0_CLIENT_ID - clientSecret: - $env: AUTH_AUTH0_CLIENT_SECRET - domain: - $env: AUTH_AUTH0_DOMAIN_ID + clientId: ${AUTH_AUTH0_CLIENT_ID} + clientSecret: ${AUTH_AUTH0_CLIENT_SECRET} + domain: ${AUTH_AUTH0_DOMAIN_ID} ``` ### 2. Create an Auth0 application in the Auth0 management console diff --git a/docs/tutorials/switching-sqlite-postgres.md b/docs/tutorials/switching-sqlite-postgres.md index 72e5589baf..a759b7dc49 100644 --- a/docs/tutorials/switching-sqlite-postgres.md +++ b/docs/tutorials/switching-sqlite-postgres.md @@ -38,14 +38,10 @@ backend: + # config options: https://node-postgres.com/api/client + client: pg + connection: -+ host: -+ $env: POSTGRES_HOST -+ port: -+ $env: POSTGRES_PORT -+ user: -+ $env: POSTGRES_USER -+ password: -+ $env: POSTGRES_PASSWORD ++ host: ${POSTGRES_HOST} ++ port: ${POSTGRES_PORT} ++ user: ${POSTGRES_USER} ++ password: ${POSTGRES_PASSWORD} + # https://node-postgres.com/features/ssl + #ssl: require # see https://www.postgresql.org/docs/current/libpq-ssl.html Table 33.1. SSL Mode Descriptions (e.g. require) + #ca: # if you have a CA file and want to verify it you can uncomment this section @@ -53,9 +49,9 @@ backend: ``` -If you have a `app-config.local.yaml` for local development, a similar update +If you have an `app-config.local.yaml` for local development, a similar update should be made there. You can set the `POSTGRES_` environment variables prior to -launching Backstage, or remove the $env keys and simply set values directly for -development. +launching Backstage, or remove the `${...}` values and simply set actual values +directly for development. The Backstage App is now ready to start up with a PostgreSQL backing database. diff --git a/microsite/package.json b/microsite/package.json index ea16fd8a32..9caafbd95b 100644 --- a/microsite/package.json +++ b/microsite/package.json @@ -15,7 +15,7 @@ "verify:sidebars": "node ./scripts/verify-sidebars" }, "devDependencies": { - "@spotify/prettier-config": "^9.0.0", + "@spotify/prettier-config": "^10.0.0", "docusaurus": "^2.0.0-alpha.70", "js-yaml": "^4.0.0", "prettier": "^2.2.1" diff --git a/microsite/yarn.lock b/microsite/yarn.lock index 898fd69ad2..5618a5ee4b 100644 --- a/microsite/yarn.lock +++ b/microsite/yarn.lock @@ -909,10 +909,10 @@ resolved "https://registry.npmjs.org/@sindresorhus/is/-/is-0.7.0.tgz#9a06f4f137ee84d7df0460c1fdb1135ffa6c50fd" integrity sha512-ONhaKPIufzzrlNbqtWFFd+jlnemX6lJAgq9ZeiZtS7I1PIf/la7CW4m83rTXRnVnsMbW2k56pGYu7AUFJD9Pow== -"@spotify/prettier-config@^9.0.0": - version "9.0.0" - resolved "https://registry.yarnpkg.com/@spotify/prettier-config/-/prettier-config-9.0.0.tgz#7b562d56573c6fc0094446fbc92b22bc318945dc" - integrity sha512-In1q0tIiqTYKAGe3KOHDcFDdZRFISyQeSeipeTHGfki23ebHRZcjxvqj5SSdBkw65D4VpSREMi0s9i5iJiMcTw== +"@spotify/prettier-config@^10.0.0": + version "10.0.0" + resolved "https://registry.yarnpkg.com/@spotify/prettier-config/-/prettier-config-10.0.0.tgz#fa076d98d2e7e6c53dd3d86a696307a7010bd056" + integrity sha512-VYOdo8P7lIScAkl02nB9KpUAuOYMManryBIBuKJkAw5D3aVtLobfmdIKvdV6MqEmGMEQPbn7w/UpnjJYhUH+IA== "@types/cheerio@^0.22.8": version "0.22.23" diff --git a/packages/backend/src/plugins/techdocs.ts b/packages/backend/src/plugins/techdocs.ts index 9883f2afbc..d0387cd15d 100644 --- a/packages/backend/src/plugins/techdocs.ts +++ b/packages/backend/src/plugins/techdocs.ts @@ -48,6 +48,9 @@ export default async function createPlugin({ discovery, }); + // checks if the publisher is working and logs the result + await publisher.getReadiness(); + // Docker client (conditionally) used by the generators, based on techdocs.generators config. const dockerClient = new Docker(); diff --git a/packages/catalog-client/src/CatalogClient.ts b/packages/catalog-client/src/CatalogClient.ts index 7426aabaac..3de25e9808 100644 --- a/packages/catalog-client/src/CatalogClient.ts +++ b/packages/catalog-client/src/CatalogClient.ts @@ -42,10 +42,14 @@ export class CatalogClient implements CatalogApi { } async getLocationById( - id: String, + id: string, options?: CatalogRequestOptions, ): Promise { - return await this.requestOptional('GET', `/locations/${id}`, options); + return await this.requestOptional( + 'GET', + `/locations/${encodeURIComponent(id)}`, + options, + ); } async getEntities( @@ -86,7 +90,9 @@ export class CatalogClient implements CatalogApi { const { kind, namespace = 'default', name } = compoundName; return this.requestOptional( 'GET', - `/entities/by-name/${kind}/${namespace}/${name}`, + `/entities/by-name/${encodeURIComponent(kind)}/${encodeURIComponent( + namespace, + )}/${encodeURIComponent(name)}`, options, ); } @@ -171,14 +177,22 @@ export class CatalogClient implements CatalogApi { id: string, options?: CatalogRequestOptions, ): Promise { - await this.requestIgnored('DELETE', `/locations/${id}`, options); + await this.requestIgnored( + 'DELETE', + `/locations/${encodeURIComponent(id)}`, + options, + ); } async removeEntityByUid( uid: string, options?: CatalogRequestOptions, ): Promise { - await this.requestIgnored('DELETE', `/entities/by-uid/${uid}`, options); + await this.requestIgnored( + 'DELETE', + `/entities/by-uid/${encodeURIComponent(uid)}`, + options, + ); } // diff --git a/packages/catalog-client/src/types.ts b/packages/catalog-client/src/types.ts index 04cbb2b689..0d25bf7483 100644 --- a/packages/catalog-client/src/types.ts +++ b/packages/catalog-client/src/types.ts @@ -46,7 +46,7 @@ export interface CatalogApi { // Locations getLocationById( - id: String, + id: string, options?: CatalogRequestOptions, ): Promise; getOriginLocationByEntity( diff --git a/packages/catalog-model/src/location/helpers.test.ts b/packages/catalog-model/src/location/helpers.test.ts index 888c79b328..3b5994b956 100644 --- a/packages/catalog-model/src/location/helpers.test.ts +++ b/packages/catalog-model/src/location/helpers.test.ts @@ -14,7 +14,11 @@ * limitations under the License. */ -import { parseLocationReference, stringifyLocationReference } from './helpers'; +import { + getEntitySourceLocation, + parseLocationReference, + stringifyLocationReference, +} from './helpers'; describe('parseLocationReference', () => { it('works for the simple case', () => { @@ -68,3 +72,48 @@ describe('stringifyLocationReference', () => { ).toThrow('Unable to stringify location reference, empty target'); }); }); + +describe('getEntitySourceLocation', () => { + it('returns the source-location', () => { + expect( + getEntitySourceLocation({ + apiVersion: 'backstage.io/v1alpha1', + kind: 'Location', + metadata: { + name: 'test', + namespace: 'default', + annotations: { + 'backstage.io/source-location': 'url:https://backstage.io/foo.yaml', + 'backstage.io/managed-by-location': 'url:https://spotify.com', + }, + }, + }), + ).toEqual({ target: 'https://backstage.io/foo.yaml', type: 'url' }); + }); + + it('returns the managed-by-location', () => { + expect( + getEntitySourceLocation({ + apiVersion: 'backstage.io/v1alpha1', + kind: 'Location', + metadata: { + name: 'test', + namespace: 'default', + annotations: { + 'backstage.io/managed-by-location': 'url:https://spotify.com', + }, + }, + }), + ).toEqual({ target: 'https://spotify.com', type: 'url' }); + }); + + it('rejects missing location annotation', () => { + expect(() => + getEntitySourceLocation({ + apiVersion: 'backstage.io/v1alpha1', + kind: 'Location', + metadata: { name: 'test', namespace: 'default' }, + }), + ).toThrow(`Entity 'location:default/test' is missing location`); + }); +}); diff --git a/packages/catalog-model/src/location/helpers.ts b/packages/catalog-model/src/location/helpers.ts index fb1be5abad..5eff598c87 100644 --- a/packages/catalog-model/src/location/helpers.ts +++ b/packages/catalog-model/src/location/helpers.ts @@ -14,6 +14,9 @@ * limitations under the License. */ +import { Entity, stringifyEntityRef } from '../entity'; +import { LOCATION_ANNOTATION, SOURCE_LOCATION_ANNOTATION } from './annotation'; + /** * Parses a string form location reference. * @@ -80,3 +83,26 @@ export function stringifyLocationReference(ref: { return `${type}:${target}`; } + +/** + * Returns the source code location of the Entity, to the extent that one exists. + * + * If the returned location type is of type 'url', the target should be readable at least + * using the UrlReader from @backstage/backend-common. If it is not of type 'url', the caller + * needs to have explicit handling of each location type or signal that it is not supported. + */ +export function getEntitySourceLocation( + entity: Entity, +): { type: string; target: string } { + const locationRef = + entity.metadata?.annotations?.[SOURCE_LOCATION_ANNOTATION] ?? + entity.metadata?.annotations?.[LOCATION_ANNOTATION]; + + if (!locationRef) { + throw new Error( + `Entity '${stringifyEntityRef(entity)}' is missing location`, + ); + } + + return parseLocationReference(locationRef); +} diff --git a/packages/catalog-model/src/location/index.ts b/packages/catalog-model/src/location/index.ts index fddc8bde37..751172c6ba 100644 --- a/packages/catalog-model/src/location/index.ts +++ b/packages/catalog-model/src/location/index.ts @@ -19,7 +19,11 @@ export { ORIGIN_LOCATION_ANNOTATION, SOURCE_LOCATION_ANNOTATION, } from './annotation'; -export { parseLocationReference, stringifyLocationReference } from './helpers'; +export { + parseLocationReference, + stringifyLocationReference, + getEntitySourceLocation, +} from './helpers'; export type { Location, LocationSpec } from './types'; export { analyzeLocationSchema, diff --git a/packages/cli/src/lib/bundler/server.ts b/packages/cli/src/lib/bundler/server.ts index 5fc3a7b5de..24a24d25d2 100644 --- a/packages/cli/src/lib/bundler/server.ts +++ b/packages/cli/src/lib/bundler/server.ts @@ -43,7 +43,7 @@ export async function serveBundle(options: ServeOptions) { const compiler = webpack(config); const server = new WebpackDevServer(compiler, { - hot: true, + hot: !process.env.CI, contentBase: paths.targetPublic, contentBasePublicPath: config.output?.publicPath, publicPath: config.output?.publicPath, diff --git a/packages/core/src/layout/Header/Header.stories.tsx b/packages/core/src/layout/Header/Header.stories.tsx index 3deb980dfd..db4ddac8ec 100644 --- a/packages/core/src/layout/Header/Header.stories.tsx +++ b/packages/core/src/layout/Header/Header.stories.tsx @@ -53,38 +53,6 @@ export const Apis = () => ( ); -export const Grpc = () => ( - -
- {labels} -
-
-); - -export const AsyncApi = () => ( - -
- {labels} -
-
-); - -export const Graphql = () => ( - -
- {labels} -
-
-); - -export const OpenApi = () => ( - -
- {labels} -
-
-); - export const Tool = () => (
diff --git a/packages/create-app/templates/default-app/app-config.yaml.hbs b/packages/create-app/templates/default-app/app-config.yaml.hbs index e7e44c9961..442e90944e 100644 --- a/packages/create-app/templates/default-app/app-config.yaml.hbs +++ b/packages/create-app/templates/default-app/app-config.yaml.hbs @@ -25,14 +25,10 @@ backend: database: client: pg connection: - host: - $env: POSTGRES_HOST - port: - $env: POSTGRES_PORT - user: - $env: POSTGRES_USER - password: - $env: POSTGRES_PASSWORD + host: ${POSTGRES_HOST} + port: ${POSTGRES_PORT} + user: ${POSTGRES_USER} + password: ${POSTGRES_PASSWORD} # https://node-postgres.com/features/ssl #ssl: require # see https://www.postgresql.org/docs/current/libpq-ssl.html Table 33.1. SSL Mode Descriptions (e.g. require) #ca: # if you have a CA file and want to verify it you can uncomment this section @@ -43,13 +39,11 @@ backend: integrations: github: - host: github.com - token: - $env: GITHUB_TOKEN + token: ${GITHUB_TOKEN} ### Example for how to add your GitHub Enterprise instance using the API: # - host: ghe.example.net # apiBaseUrl: https://ghe.example.net/api/v3 - # token: - # $env: GHE_TOKEN + # token: ${GHE_TOKEN} proxy: '/test': @@ -73,8 +67,7 @@ auth: scaffolder: github: - token: - $env: GITHUB_TOKEN + token: ${GITHUB_TOKEN} visibility: public # or 'internal' or 'private' catalog: diff --git a/packages/create-app/templates/default-app/packages/backend/src/plugins/techdocs.ts b/packages/create-app/templates/default-app/packages/backend/src/plugins/techdocs.ts index 0ba08cefaf..231a7e7fd7 100644 --- a/packages/create-app/templates/default-app/packages/backend/src/plugins/techdocs.ts +++ b/packages/create-app/templates/default-app/packages/backend/src/plugins/techdocs.ts @@ -34,6 +34,9 @@ export default async function createPlugin({ discovery, }); + // checks if the publisher is working and logs the result + await publisher.getReadiness(); + // Docker client (conditionally) used by the generators, based on techdocs.generators config. const dockerClient = new Docker(); diff --git a/packages/techdocs-common/__mocks__/@google-cloud/storage.ts b/packages/techdocs-common/__mocks__/@google-cloud/storage.ts index 264852d31c..f8d0368f27 100644 --- a/packages/techdocs-common/__mocks__/@google-cloud/storage.ts +++ b/packages/techdocs-common/__mocks__/@google-cloud/storage.ts @@ -81,10 +81,12 @@ class Bucket { this.bucketName = bucketName; } - getMetadata() { - return new Promise(resolve => { - resolve(''); - }); + async getMetadata() { + if (this.bucketName === 'errorBucket') { + throw Error('Bucket does not exist'); + } + + return ''; } upload(source: string, { destination }) { diff --git a/packages/techdocs-common/__mocks__/aws-sdk.ts b/packages/techdocs-common/__mocks__/aws-sdk.ts index 363662c63c..eb15a218ab 100644 --- a/packages/techdocs-common/__mocks__/aws-sdk.ts +++ b/packages/techdocs-common/__mocks__/aws-sdk.ts @@ -80,10 +80,16 @@ export class S3 { }; } - headBucket() { - return new Promise(resolve => { - resolve(''); - }); + headBucket({ Bucket }) { + return { + promise: async () => { + if (Bucket === 'errorBucket') { + throw new Error('Bucket does not exist'); + } + + return {}; + }, + }; } upload({ Key }: { Key: string }) { diff --git a/packages/techdocs-common/__mocks__/pkgcloud.ts b/packages/techdocs-common/__mocks__/pkgcloud.ts index 5d9f81cda0..dca86238e4 100644 --- a/packages/techdocs-common/__mocks__/pkgcloud.ts +++ b/packages/techdocs-common/__mocks__/pkgcloud.ts @@ -13,10 +13,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +import { EventEmitter } from 'events'; import fs from 'fs-extra'; import os from 'os'; import path from 'path'; -import { EventEmitter } from 'events'; +import { ClientError } from 'pkgcloud'; const rootDir = os.platform() === 'win32' ? 'C:\\rootDir' : '/rootDir'; @@ -37,12 +38,11 @@ class PkgCloudStorageClient { getFile( containerName: string, file: string, - callback: (err: any, file: string) => any, + callback: (err: any, file: any) => any, ) { checkFileExists(file).then(res => { if (!res) { - callback('File does not exist', file); - throw new Error('File does not exist'); + callback('File does not exist', undefined); } else { callback(undefined, 'success'); } @@ -51,13 +51,12 @@ class PkgCloudStorageClient { getContainer( containerName: string, - callback: (err: string, container: string) => any, + callback: (err: ClientError, container: any) => any, ) { if (containerName !== 'mock') { - callback('Container does not exist', containerName); - throw new Error('Container does not exist'); + callback(new Error('Container does not exist'), undefined); } else { - callback('Container does not exist', 'success'); + callback(undefined, 'success'); } } diff --git a/packages/techdocs-common/src/stages/publish/awsS3.test.ts b/packages/techdocs-common/src/stages/publish/awsS3.test.ts index d3281c2645..15da3b7cfc 100644 --- a/packages/techdocs-common/src/stages/publish/awsS3.test.ts +++ b/packages/techdocs-common/src/stages/publish/awsS3.test.ts @@ -13,16 +13,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +import { getVoidLogger } from '@backstage/backend-common'; import { Entity, - EntityName, ENTITY_DEFAULT_NAMESPACE, + EntityName, } from '@backstage/catalog-model'; import { ConfigReader } from '@backstage/config'; import mockFs from 'mock-fs'; import os from 'os'; import path from 'path'; -import * as winston from 'winston'; import { AwsS3Publish } from './awsS3'; import { PublisherBase, TechDocsMetadata } from './types'; @@ -59,9 +59,7 @@ const getEntityRootDir = (entity: Entity) => { return path.join(rootDir, namespace || ENTITY_DEFAULT_NAMESPACE, kind, name); }; -const logger = winston.createLogger(); -jest.spyOn(logger, 'info').mockReturnValue(logger); -jest.spyOn(logger, 'error').mockReturnValue(logger); +const logger = getVoidLogger(); let publisher: PublisherBase; @@ -87,6 +85,39 @@ beforeEach(() => { }); describe('AwsS3Publish', () => { + describe('getReadiness', () => { + it('should validate correct config', async () => { + expect(await publisher.getReadiness()).toEqual({ + isAvailable: true, + }); + }); + + it('should reject incorrect config', async () => { + const mockConfig = new ConfigReader({ + techdocs: { + requestUrl: 'http://localhost:7000', + publisher: { + type: 'awsS3', + awsS3: { + credentials: { + accessKeyId: 'accessKeyId', + secretAccessKey: 'secretAccessKey', + }, + // this bucket name will throw an error + bucketName: 'errorBucket', + }, + }, + }, + }); + + const errorPublisher = AwsS3Publish.fromConfig(mockConfig, logger); + + expect(await errorPublisher.getReadiness()).toEqual({ + isAvailable: false, + }); + }); + }); + describe('publish', () => { beforeEach(() => { const entity = createMockEntity(); diff --git a/packages/techdocs-common/src/stages/publish/awsS3.ts b/packages/techdocs-common/src/stages/publish/awsS3.ts index c4c39898f4..9c6684fdca 100644 --- a/packages/techdocs-common/src/stages/publish/awsS3.ts +++ b/packages/techdocs-common/src/stages/publish/awsS3.ts @@ -17,16 +17,21 @@ import { Entity, EntityName } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; import aws, { Credentials } from 'aws-sdk'; import { ManagedUpload } from 'aws-sdk/clients/s3'; +import { CredentialsOptions } from 'aws-sdk/lib/credentials'; import express from 'express'; import fs from 'fs-extra'; import JSON5 from 'json5'; import createLimiter from 'p-limit'; -import { CredentialsOptions } from 'aws-sdk/lib/credentials'; import path from 'path'; import { Readable } from 'stream'; import { Logger } from 'winston'; import { getFileTreeRecursively, getHeadersForFileExtension } from './helpers'; -import { PublisherBase, PublishRequest, TechDocsMetadata } from './types'; +import { + PublisherBase, + PublishRequest, + ReadinessResponse, + TechDocsMetadata, +} from './types'; const streamToBuffer = (stream: Readable): Promise => { return new Promise((resolve, reject) => { @@ -81,30 +86,6 @@ export class AwsS3Publish implements PublisherBase { ...(endpoint && { endpoint }), }); - // Check if the defined bucket exists. Being able to connect means the configuration is good - // and the storage client will work. - storageClient.headBucket( - { - Bucket: bucketName, - }, - err => { - if (err) { - logger.error( - `Could not retrieve metadata about the AWS S3 bucket ${bucketName}. ` + - 'Make sure the bucket exists. Also make sure that authentication is setup either by ' + - 'explicitly defining credentials and region in techdocs.publisher.awsS3 in app config or ' + - 'by using environment variables. Refer to https://backstage.io/docs/features/techdocs/using-cloud-storage', - ); - logger.error(`from AWS client library: ${err.message}`); - throw new Error(); - } else { - logger.info( - `Successfully connected to the AWS S3 bucket ${bucketName}.`, - ); - } - }, - ); - return new AwsS3Publish(storageClient, bucketName, logger); } @@ -149,6 +130,35 @@ export class AwsS3Publish implements PublisherBase { this.logger = logger; } + /** + * Check if the defined bucket exists. Being able to connect means the configuration is good + * and the storage client will work. + */ + async getReadiness(): Promise { + try { + await this.storageClient + .headBucket({ Bucket: this.bucketName }) + .promise(); + + this.logger.info( + `Successfully connected to the AWS S3 bucket ${this.bucketName}.`, + ); + + return { isAvailable: true }; + } catch (error) { + this.logger.error( + `Could not retrieve metadata about the AWS S3 bucket ${this.bucketName}. ` + + 'Make sure the bucket exists. Also make sure that authentication is setup either by ' + + 'explicitly defining credentials and region in techdocs.publisher.awsS3 in app config or ' + + 'by using environment variables. Refer to https://backstage.io/docs/features/techdocs/using-cloud-storage', + ); + this.logger.error(`from AWS client library`, error); + return { + isAvailable: false, + }; + } + } + /** * Upload all the files from the generated `directory` to the S3 bucket. * Directory structure used in the bucket is - entityNamespace/entityKind/entityName/index.html diff --git a/packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts b/packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts index 9442fb0e37..c431e924a8 100644 --- a/packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts +++ b/packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts @@ -16,8 +16,8 @@ import { getVoidLogger } from '@backstage/backend-common'; import { Entity, - EntityName, ENTITY_DEFAULT_NAMESPACE, + EntityName, } from '@backstage/catalog-model'; import { ConfigReader } from '@backstage/config'; import mockFs from 'mock-fs'; @@ -59,12 +59,9 @@ const getEntityRootDir = (entity: Entity) => { return path.join(rootDir, namespace || ENTITY_DEFAULT_NAMESPACE, kind, name); }; -function createLogger() { - const logger = getVoidLogger(); - jest.spyOn(logger, 'info').mockReturnValue(logger); - jest.spyOn(logger, 'error').mockReturnValue(logger); - return logger; -} +const logger = getVoidLogger(); +jest.spyOn(logger, 'info').mockReturnValue(logger); +jest.spyOn(logger, 'error').mockReturnValue(logger); let publisher: PublisherBase; beforeEach(async () => { @@ -85,13 +82,51 @@ beforeEach(async () => { }, }); - publisher = await AzureBlobStoragePublish.fromConfig( - mockConfig, - createLogger(), - ); + publisher = AzureBlobStoragePublish.fromConfig(mockConfig, logger); }); describe('publishing with valid credentials', () => { + describe('getReadiness', () => { + it('should validate correct config', async () => { + expect(await publisher.getReadiness()).toEqual({ + isAvailable: true, + }); + }); + + it('should reject incorrect config', async () => { + const mockConfig = new ConfigReader({ + techdocs: { + requestUrl: 'http://localhost:7000', + publisher: { + type: 'azureBlobStorage', + azureBlobStorage: { + credentials: { + accountName: 'accountName', + accountKey: 'accountKey', + }, + containerName: 'bad_container', + }, + }, + }, + }); + + const errorPublisher = await AzureBlobStoragePublish.fromConfig( + mockConfig, + logger, + ); + + expect(await errorPublisher.getReadiness()).toEqual({ + isAvailable: false, + }); + + expect(logger.error).toHaveBeenCalledWith( + expect.stringContaining( + `Could not retrieve metadata about the Azure Blob Storage container bad_container.`, + ), + ); + }); + }); + describe('publish', () => { beforeEach(() => { const entity = createMockEntity(); @@ -151,6 +186,60 @@ describe('publishing with valid credentials', () => { }); mockFs.restore(); }); + + it('reports an error when bad account credentials', async () => { + const mockConfig = new ConfigReader({ + techdocs: { + requestUrl: 'http://localhost:7000', + publisher: { + type: 'azureBlobStorage', + azureBlobStorage: { + credentials: { + accountName: 'failupload', + accountKey: 'accountKey', + }, + containerName: 'containerName', + }, + }, + }, + }); + + publisher = await AzureBlobStoragePublish.fromConfig(mockConfig, logger); + + const entity = createMockEntity(); + const entityRootDir = getEntityRootDir(entity); + + mockFs({ + [entityRootDir]: { + 'index.html': '', + }, + }); + + let error; + try { + await publisher.publish({ + entity, + directory: entityRootDir, + }); + } catch (e) { + error = e; + } + + expect(error.message).toContain( + `Unable to upload file(s) to Azure Blob Storage.`, + ); + + expect(logger.error).toHaveBeenCalledWith( + expect.stringContaining( + `Unable to upload file(s) to Azure Blob Storage. Error: Upload failed for ${path.join( + entityRootDir, + 'index.html', + )} with status code 500`, + ), + ); + + mockFs.restore(); + }); }); describe('hasDocsBeenGenerated', () => { @@ -243,156 +332,3 @@ describe('publishing with valid credentials', () => { }); }); }); - -describe('error reporting', () => { - it('reports an error when unable to read container properties', async () => { - const mockConfig = new ConfigReader({ - techdocs: { - requestUrl: 'http://localhost:7000', - publisher: { - type: 'azureBlobStorage', - azureBlobStorage: { - credentials: { - accountName: 'accountName', - }, - containerName: 'bad_container', - }, - }, - }, - }); - - const logger = createLogger(); - - let error; - try { - publisher = await AzureBlobStoragePublish.fromConfig(mockConfig, logger); - } catch (e) { - error = e; - } - - expect(error).toBeInstanceOf(Error); - - expect(logger.error).toHaveBeenCalledWith( - expect.stringContaining( - `Could not retrieve metadata about the Azure Blob Storage container bad_container.`, - ), - ); - }); - - it('reports an error when bad account credentials', async () => { - const mockConfig = new ConfigReader({ - techdocs: { - requestUrl: 'http://localhost:7000', - publisher: { - type: 'azureBlobStorage', - azureBlobStorage: { - credentials: { - accountName: 'failupload', - accountKey: 'accountKey', - }, - containerName: 'containerName', - }, - }, - }, - }); - - const logger = createLogger(); - - publisher = await AzureBlobStoragePublish.fromConfig(mockConfig, logger); - - const entity = createMockEntity(); - const entityRootDir = getEntityRootDir(entity); - - mockFs({ - [entityRootDir]: { - 'index.html': '', - }, - }); - - let error; - try { - await publisher.publish({ - entity, - directory: entityRootDir, - }); - } catch (e) { - error = e; - } - - expect(error.message).toContain( - `Unable to upload file(s) to Azure Blob Storage.`, - ); - - expect(logger.error).toHaveBeenCalledWith( - expect.stringContaining( - `Unable to upload file(s) to Azure Blob Storage. Error: Upload failed for ${path.join( - entityRootDir, - 'index.html', - )} with status code 500`, - ), - ); - - mockFs.restore(); - }); - - describe('fetchTechDocsMetadata', () => { - it('should return tech docs metadata', async () => { - const entityNameMock = createMockEntityName(); - const entity = createMockEntity(); - const entityRootDir = getEntityRootDir(entity); - - mockFs({ - [entityRootDir]: { - 'techdocs_metadata.json': - '{"site_name": "backstage", "site_description": "site_content", "etag": "etag"}', - }, - }); - const expectedMetadata: TechDocsMetadata = { - site_name: 'backstage', - site_description: 'site_content', - etag: 'etag', - }; - expect( - await publisher.fetchTechDocsMetadata(entityNameMock), - ).toStrictEqual(expectedMetadata); - mockFs.restore(); - }); - - it('should return tech docs metadata when json encoded with single quotes', async () => { - const entityNameMock = createMockEntityName(); - const entity = createMockEntity(); - const entityRootDir = getEntityRootDir(entity); - - mockFs({ - [entityRootDir]: { - 'techdocs_metadata.json': `{'site_name': 'backstage', 'site_description': 'site_content', 'etag': 'etag'}`, - }, - }); - - const expectedMetadata: TechDocsMetadata = { - site_name: 'backstage', - site_description: 'site_content', - etag: 'etag', - }; - expect( - await publisher.fetchTechDocsMetadata(entityNameMock), - ).toStrictEqual(expectedMetadata); - mockFs.restore(); - }); - - it('should return an error if the techdocs_metadata.json file is not present', async () => { - const entityNameMock = createMockEntityName(); - - let error; - try { - await publisher.fetchTechDocsMetadata(entityNameMock); - } catch (e) { - error = e; - } - - expect(error.message).toEqual( - expect.stringContaining('TechDocs metadata fetch'), - ); - }); - }); -}); diff --git a/packages/techdocs-common/src/stages/publish/azureBlobStorage.ts b/packages/techdocs-common/src/stages/publish/azureBlobStorage.ts index 155d14fbb5..1a823c3d19 100644 --- a/packages/techdocs-common/src/stages/publish/azureBlobStorage.ts +++ b/packages/techdocs-common/src/stages/publish/azureBlobStorage.ts @@ -26,16 +26,18 @@ import limiterFactory from 'p-limit'; import { default as path, default as platformPath } from 'path'; import { Logger } from 'winston'; import { getFileTreeRecursively, getHeadersForFileExtension } from './helpers'; -import { PublisherBase, PublishRequest, TechDocsMetadata } from './types'; +import { + PublisherBase, + PublishRequest, + ReadinessResponse, + TechDocsMetadata, +} from './types'; // The number of batches that may be ongoing at the same time. const BATCH_CONCURRENCY = 3; export class AzureBlobStoragePublish implements PublisherBase { - static async fromConfig( - config: Config, - logger: Logger, - ): Promise { + static fromConfig(config: Config, logger: Logger): PublisherBase { let containerName = ''; try { containerName = config.getString( @@ -78,26 +80,6 @@ export class AzureBlobStoragePublish implements PublisherBase { credential, ); - try { - const response = await storageClient - .getContainerClient(containerName) - .getProperties(); - - if (response._response.status >= 400) { - throw new Error( - `Failed to retrieve metadata from ${response._response.request.url} with status code ${response._response.status}.`, - ); - } - } catch (e) { - logger.error( - `Could not retrieve metadata about the Azure Blob Storage container ${containerName}. ` + - 'Make sure that the Azure project and container exist and the access key is setup correctly ' + - 'techdocs.publisher.azureBlobStorage.credentials defined in app config has correct permissions. ' + - 'Refer to https://backstage.io/docs/features/techdocs/using-cloud-storage', - ); - throw new Error(`from Azure Blob Storage client library: ${e.message}`); - } - return new AzureBlobStoragePublish(storageClient, containerName, logger); } @@ -111,6 +93,37 @@ export class AzureBlobStoragePublish implements PublisherBase { this.logger = logger; } + async getReadiness(): Promise { + try { + const response = await this.storageClient + .getContainerClient(this.containerName) + .getProperties(); + + if (response._response.status === 200) { + return { + isAvailable: true, + }; + } + + if (response._response.status >= 400) { + this.logger.error( + `Failed to retrieve metadata from ${response._response.request.url} with status code ${response._response.status}.`, + ); + } + } catch (e) { + this.logger.error(`from Azure Blob Storage client library: ${e.message}`); + } + + this.logger.error( + `Could not retrieve metadata about the Azure Blob Storage container ${this.containerName}. ` + + 'Make sure that the Azure project and container exist and the access key is setup correctly ' + + 'techdocs.publisher.azureBlobStorage.credentials defined in app config has correct permissions. ' + + 'Refer to https://backstage.io/docs/features/techdocs/using-cloud-storage', + ); + + return { isAvailable: false }; + } + /** * Upload all the files from the generated `directory` to the Azure Blob Storage container. * Directory structure used in the container is - entityNamespace/entityKind/entityName/index.html diff --git a/packages/techdocs-common/src/stages/publish/googleStorage.test.ts b/packages/techdocs-common/src/stages/publish/googleStorage.test.ts index 07bc5c06b9..7e580b16bc 100644 --- a/packages/techdocs-common/src/stages/publish/googleStorage.test.ts +++ b/packages/techdocs-common/src/stages/publish/googleStorage.test.ts @@ -16,8 +16,8 @@ import { getVoidLogger } from '@backstage/backend-common'; import { Entity, - EntityName, ENTITY_DEFAULT_NAMESPACE, + EntityName, } from '@backstage/catalog-model'; import { ConfigReader } from '@backstage/config'; import mockFs from 'mock-fs'; @@ -83,6 +83,35 @@ beforeEach(async () => { }); describe('GoogleGCSPublish', () => { + describe('getReadiness', () => { + it('should validate correct config', async () => { + expect(await publisher.getReadiness()).toEqual({ + isAvailable: true, + }); + }); + + it('should reject incorrect config', async () => { + const mockConfig = new ConfigReader({ + techdocs: { + requestUrl: 'http://localhost:7000', + publisher: { + type: 'googleGcs', + googleGcs: { + credentials: '{}', + bucketName: 'errorBucket', + }, + }, + }, + }); + + const errorPublisher = GoogleGCSPublish.fromConfig(mockConfig, logger); + + expect(await errorPublisher.getReadiness()).toEqual({ + isAvailable: false, + }); + }); + }); + describe('publish', () => { beforeEach(() => { const entity = createMockEntity(); diff --git a/packages/techdocs-common/src/stages/publish/googleStorage.ts b/packages/techdocs-common/src/stages/publish/googleStorage.ts index 12ecf45b2d..28fbb42ece 100644 --- a/packages/techdocs-common/src/stages/publish/googleStorage.ts +++ b/packages/techdocs-common/src/stages/publish/googleStorage.ts @@ -26,13 +26,15 @@ import createLimiter from 'p-limit'; import path from 'path'; import { Logger } from 'winston'; import { getFileTreeRecursively, getHeadersForFileExtension } from './helpers'; -import { PublisherBase, PublishRequest, TechDocsMetadata } from './types'; +import { + PublisherBase, + PublishRequest, + ReadinessResponse, + TechDocsMetadata, +} from './types'; export class GoogleGCSPublish implements PublisherBase { - static async fromConfig( - config: Config, - logger: Logger, - ): Promise { + static fromConfig(config: Config, logger: Logger): PublisherBase { let bucketName = ''; try { bucketName = config.getString('techdocs.publisher.googleGcs.bucketName'); @@ -65,21 +67,6 @@ export class GoogleGCSPublish implements PublisherBase { }), }); - // Check if the defined bucket exists. Being able to connect means the configuration is good - // and the storage client will work. - try { - await storageClient.bucket(bucketName).getMetadata(); - logger.info(`Successfully connected to the GCS bucket ${bucketName}.`); - } catch (err) { - logger.error( - `Could not retrieve metadata about the GCS bucket ${bucketName}. ` + - 'Make sure the bucket exists. Also make sure that authentication is setup either by explicitly defining ' + - 'techdocs.publisher.googleGcs.credentials in app config or by using environment variables. ' + - 'Refer to https://backstage.io/docs/features/techdocs/using-cloud-storage', - ); - throw new Error(err.message); - } - return new GoogleGCSPublish(storageClient, bucketName, logger); } @@ -93,6 +80,33 @@ export class GoogleGCSPublish implements PublisherBase { this.logger = logger; } + /** + * Check if the defined bucket exists. Being able to connect means the configuration is good + * and the storage client will work. + */ + async getReadiness(): Promise { + try { + await this.storageClient.bucket(this.bucketName).getMetadata(); + this.logger.info( + `Successfully connected to the GCS bucket ${this.bucketName}.`, + ); + + return { + isAvailable: true, + }; + } catch (err) { + this.logger.error( + `Could not retrieve metadata about the GCS bucket ${this.bucketName}. ` + + 'Make sure the bucket exists. Also make sure that authentication is setup either by explicitly defining ' + + 'techdocs.publisher.googleGcs.credentials in app config or by using environment variables. ' + + 'Refer to https://backstage.io/docs/features/techdocs/using-cloud-storage', + ); + this.logger.error(`from GCS client library: ${err.message}`); + + return { isAvailable: false }; + } + } + /** * Upload all the files from the generated `directory` to the GCS bucket. * Directory structure used in the bucket is - entityNamespace/entityKind/entityName/index.html diff --git a/packages/techdocs-common/src/stages/publish/local.ts b/packages/techdocs-common/src/stages/publish/local.ts index 3088c1a695..e09473cf60 100644 --- a/packages/techdocs-common/src/stages/publish/local.ts +++ b/packages/techdocs-common/src/stages/publish/local.ts @@ -28,6 +28,7 @@ import { PublisherBase, PublishRequest, PublishResponse, + ReadinessResponse, TechDocsMetadata, } from './types'; @@ -65,6 +66,12 @@ export class LocalPublish implements PublisherBase { this.discovery = discovery; } + async getReadiness(): Promise { + return { + isAvailable: true, + }; + } + publish({ entity, directory }: PublishRequest): Promise { const entityNamespace = entity.metadata.namespace ?? 'default'; diff --git a/packages/techdocs-common/src/stages/publish/openStackSwift.test.ts b/packages/techdocs-common/src/stages/publish/openStackSwift.test.ts index f3b788916f..6f0aacede0 100644 --- a/packages/techdocs-common/src/stages/publish/openStackSwift.test.ts +++ b/packages/techdocs-common/src/stages/publish/openStackSwift.test.ts @@ -13,16 +13,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +import { getVoidLogger } from '@backstage/backend-common'; import { Entity, - EntityName, ENTITY_DEFAULT_NAMESPACE, + EntityName, } from '@backstage/catalog-model'; import { ConfigReader } from '@backstage/config'; import mockFs from 'mock-fs'; import os from 'os'; import path from 'path'; -import * as winston from 'winston'; import { OpenStackSwiftPublish } from './openStackSwift'; import { PublisherBase, TechDocsMetadata } from './types'; @@ -59,9 +59,7 @@ const getEntityRootDir = (entity: Entity) => { return path.join(rootDir, namespace || ENTITY_DEFAULT_NAMESPACE, kind, name); }; -const logger = winston.createLogger(); -jest.spyOn(logger, 'info').mockReturnValue(logger); -jest.spyOn(logger, 'error').mockReturnValue(logger); +const logger = getVoidLogger(); let publisher: PublisherBase; @@ -89,6 +87,43 @@ beforeEach(() => { }); describe('OpenStackSwiftPublish', () => { + describe('getReadiness', () => { + it('should validate correct config', async () => { + expect(await publisher.getReadiness()).toEqual({ + isAvailable: true, + }); + }); + + it('should reject incorrect config', async () => { + const mockConfig = new ConfigReader({ + techdocs: { + requestUrl: 'http://localhost:7000', + publisher: { + type: 'openStackSwift', + openStackSwift: { + credentials: { + username: 'mockuser', + password: 'verystrongpass', + }, + authUrl: 'mockauthurl', + region: 'mockregion', + containerName: 'errorBucket', + }, + }, + }, + }); + + const errorPublisher = OpenStackSwiftPublish.fromConfig( + mockConfig, + logger, + ); + + expect(await errorPublisher.getReadiness()).toEqual({ + isAvailable: false, + }); + }); + }); + describe('publish', () => { beforeEach(() => { const entity = createMockEntity(); diff --git a/packages/techdocs-common/src/stages/publish/openStackSwift.ts b/packages/techdocs-common/src/stages/publish/openStackSwift.ts index 0e60bc5796..bff0055402 100644 --- a/packages/techdocs-common/src/stages/publish/openStackSwift.ts +++ b/packages/techdocs-common/src/stages/publish/openStackSwift.ts @@ -15,16 +15,21 @@ */ import { Entity, EntityName } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; -import { storage } from 'pkgcloud'; import express from 'express'; import fs from 'fs-extra'; import JSON5 from 'json5'; import createLimiter from 'p-limit'; import path from 'path'; +import { storage } from 'pkgcloud'; import { Readable } from 'stream'; import { Logger } from 'winston'; import { getFileTreeRecursively, getHeadersForFileExtension } from './helpers'; -import { PublisherBase, PublishRequest, TechDocsMetadata } from './types'; +import { + PublisherBase, + PublishRequest, + ReadinessResponse, + TechDocsMetadata, +} from './types'; const streamToBuffer = (stream: Readable): Promise => { return new Promise((resolve, reject) => { @@ -70,25 +75,6 @@ export class OpenStackSwiftPublish implements PublisherBase { region: openStackSwiftConfig.getString('region'), }); - // Check if the defined container exists. Being able to connect means the configuration is good - // and the storage client will work. - storageClient.getContainer(containerName, (err, container) => { - if (container) { - logger.info( - `Successfully connected to the OpenStack Swift container ${containerName}.`, - ); - } else { - logger.error( - `Could not retrieve metadata about the OpenStack Swift container ${containerName}. ` + - 'Make sure the container exists. Also make sure that authentication is setup either by ' + - 'explicitly defining credentials and region in techdocs.publisher.openStackSwift in app config or ' + - 'by using environment variables. Refer to https://backstage.io/docs/features/techdocs/using-cloud-storage', - ); - - logger.error(`from OpenStack client library: ${err.message}`); - } - }); - return new OpenStackSwiftPublish(storageClient, containerName, logger); } @@ -102,6 +88,37 @@ export class OpenStackSwiftPublish implements PublisherBase { this.logger = logger; } + /* + * Check if the defined container exists. Being able to connect means the configuration is good + * and the storage client will work. + */ + getReadiness(): Promise { + return new Promise(resolve => { + this.storageClient.getContainer(this.containerName, (err, container) => { + if (container) { + this.logger.info( + `Successfully connected to the OpenStack Swift container ${this.containerName}.`, + ); + resolve({ + isAvailable: true, + }); + } else { + this.logger.error( + `Could not retrieve metadata about the OpenStack Swift container ${this.containerName}. ` + + 'Make sure the container exists. Also make sure that authentication is setup either by ' + + 'explicitly defining credentials and region in techdocs.publisher.openStackSwift in app config or ' + + 'by using environment variables. Refer to https://backstage.io/docs/features/techdocs/using-cloud-storage', + ); + + this.logger.error(`from OpenStack client library: ${err.message}`); + resolve({ + isAvailable: false, + }); + } + }); + }); + } + /** * Upload all the files from the generated `directory` to the OpenStack Swift container. * Directory structure used in the bucket is - entityNamespace/entityKind/entityName/index.html diff --git a/packages/techdocs-common/src/stages/publish/publish.ts b/packages/techdocs-common/src/stages/publish/publish.ts index f70a5f7626..909a4092ac 100644 --- a/packages/techdocs-common/src/stages/publish/publish.ts +++ b/packages/techdocs-common/src/stages/publish/publish.ts @@ -13,16 +13,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { Logger } from 'winston'; -import { Config } from '@backstage/config'; -import { PluginEndpointDiscovery } from '@backstage/backend-common'; -import { PublisherType, PublisherBase } from './types'; -import { LocalPublish } from './local'; -import { GoogleGCSPublish } from './googleStorage'; +import { PluginEndpointDiscovery } from '@backstage/backend-common'; +import { Config } from '@backstage/config'; +import { Logger } from 'winston'; import { AwsS3Publish } from './awsS3'; import { AzureBlobStoragePublish } from './azureBlobStorage'; +import { GoogleGCSPublish } from './googleStorage'; +import { LocalPublish } from './local'; import { OpenStackSwiftPublish } from './openStackSwift'; +import { PublisherBase, PublisherType } from './types'; type factoryOptions = { logger: Logger; @@ -45,7 +45,7 @@ export class Publisher { switch (publisherType) { case 'googleGcs': logger.info('Creating Google Storage Bucket publisher for TechDocs'); - return await GoogleGCSPublish.fromConfig(config, logger); + return GoogleGCSPublish.fromConfig(config, logger); case 'awsS3': logger.info('Creating AWS S3 Bucket publisher for TechDocs'); return AwsS3Publish.fromConfig(config, logger); diff --git a/packages/techdocs-common/src/stages/publish/types.ts b/packages/techdocs-common/src/stages/publish/types.ts index 83fee55133..c2849b8e35 100644 --- a/packages/techdocs-common/src/stages/publish/types.ts +++ b/packages/techdocs-common/src/stages/publish/types.ts @@ -37,6 +37,14 @@ export type PublishResponse = { remoteUrl?: string; } | void; +/** + * Result for the validation check. + */ +export type ReadinessResponse = { + /** If true, the publisher is able to interact with the backing storage. */ + isAvailable: boolean; +}; + /** * Type to hold metadata found in techdocs_metadata.json and associated with each site * @param etag ETag of the resource used to generate the site. Usually the latest commit sha of the source repository. @@ -53,6 +61,14 @@ export type TechDocsMetadata = { * It also provides APIs to communicate with the storage service. */ export interface PublisherBase { + /** + * Check if the publisher is ready. This check tries to perform certain checks to see if the + * publisher is configured correctly and can be used to publish or read documentations. + * The different implementations might e.g. use the provided service credentials to access the + * target or check if a folder/bucket is available. + */ + getReadiness(): Promise; + /** * Store the generated static files onto a storage service (either local filesystem or external service). * diff --git a/packages/theme/src/pageTheme.ts b/packages/theme/src/pageTheme.ts index c9a61bd95b..36f49bcdf9 100644 --- a/packages/theme/src/pageTheme.ts +++ b/packages/theme/src/pageTheme.ts @@ -63,5 +63,5 @@ export const pageTheme: Record = { library: genPageTheme(colorVariants.rubyRed, shapes.wave), other: genPageTheme(colorVariants.darkGrey, shapes.wave), app: genPageTheme(colorVariants.toastyOrange, shapes.wave), - apis: genPageTheme(colorVariants.eveningSea, shapes.wave2), + apis: genPageTheme(colorVariants.teal, shapes.wave2), }; diff --git a/plugins/bitrise/README.md b/plugins/bitrise/README.md index 2bff1f0d74..59d85b0a98 100644 --- a/plugins/bitrise/README.md +++ b/plugins/bitrise/README.md @@ -55,8 +55,7 @@ proxy: target: 'https://api.bitrise.io/v0.1' allowedMethods: ['GET'] headers: - Authorization: - $env: BITRISE_AUTH_TOKEN + Authorization: ${BITRISE_AUTH_TOKEN} ``` Learn on https://devcenter.bitrise.io/api/authentication how to create a new Bitrise token. diff --git a/plugins/catalog/src/CatalogClientWrapper.ts b/plugins/catalog/src/CatalogClientWrapper.ts index 32dff8a0a1..4966d9c13f 100644 --- a/plugins/catalog/src/CatalogClientWrapper.ts +++ b/plugins/catalog/src/CatalogClientWrapper.ts @@ -42,7 +42,7 @@ export class CatalogClientWrapper implements CatalogApi { } async getLocationById( - id: String, + id: string, options?: CatalogRequestOptions, ): Promise { return await this.client.getLocationById(id, { diff --git a/plugins/circleci/README.md b/plugins/circleci/README.md index 35fed49827..d30e950aa9 100644 --- a/plugins/circleci/README.md +++ b/plugins/circleci/README.md @@ -43,8 +43,7 @@ proxy: '/circleci/api': target: https://circleci.com/api/v1.1 headers: - Circle-Token: - $env: CIRCLECI_AUTH_TOKEN + Circle-Token: ${CIRCLECI_AUTH_TOKEN} ``` 5. Get and provide `CIRCLECI_AUTH_TOKEN` as env variable (https://circleci.com/docs/api/#add-an-api-token) diff --git a/plugins/fossa/README.md b/plugins/fossa/README.md index 6e53ced0cf..722d8d2cf3 100644 --- a/plugins/fossa/README.md +++ b/plugins/fossa/README.md @@ -50,9 +50,7 @@ proxy: target: https://app.fossa.io/api allowedMethods: ['GET'] headers: - Authorization: - # Content: 'token ' - $env: FOSSA_AUTH_HEADER + Authorization: token ${FOSSA_API_TOKEN} # if you have a fossa organization, configure your id here fossa: diff --git a/plugins/jenkins/README.md b/plugins/jenkins/README.md index 13ff9b3547..b3bc21b7d5 100644 --- a/plugins/jenkins/README.md +++ b/plugins/jenkins/README.md @@ -29,15 +29,13 @@ proxy: target: 'http://localhost:8080' # your Jenkins URL changeOrigin: true headers: - Authorization: - $env: JENKINS_BASIC_AUTH_HEADER + Authorization: Basic ${JENKINS_BASIC_AUTH_HEADER} ``` 4. Add an environment variable which contains the Jenkins credentials, (note: use an API token not your password). Here user is the name of the user created in Jenkins. ```shell -HEADER=$(echo -n user:api-token | base64) -export JENKINS_BASIC_AUTH_HEADER="Basic $HEADER" +export JENKINS_BASIC_AUTH_HEADER=$(echo -n user:api-token | base64) ``` 5. Run app with `yarn start` diff --git a/plugins/kubernetes-backend/src/kubernetes-auth-translator/KubernetesAuthTranslatorGenerator.ts b/plugins/kubernetes-backend/src/kubernetes-auth-translator/KubernetesAuthTranslatorGenerator.ts index 4ae2aff35a..d222bdbce1 100644 --- a/plugins/kubernetes-backend/src/kubernetes-auth-translator/KubernetesAuthTranslatorGenerator.ts +++ b/plugins/kubernetes-backend/src/kubernetes-auth-translator/KubernetesAuthTranslatorGenerator.ts @@ -21,7 +21,7 @@ import { AwsIamKubernetesAuthTranslator } from './AwsIamKubernetesAuthTranslator export class KubernetesAuthTranslatorGenerator { static getKubernetesAuthTranslatorInstance( - authProvider: String, + authProvider: string, ): KubernetesAuthTranslator { switch (authProvider) { case 'google': { diff --git a/plugins/kubernetes-backend/src/service/KubernetesFanOutHandler.test.ts b/plugins/kubernetes-backend/src/service/KubernetesFanOutHandler.test.ts index e76a114d54..7ec1a317d8 100644 --- a/plugins/kubernetes-backend/src/service/KubernetesFanOutHandler.test.ts +++ b/plugins/kubernetes-backend/src/service/KubernetesFanOutHandler.test.ts @@ -34,8 +34,8 @@ const mockFetch = (mock: jest.Mock) => { }; function generateMockResourcesAndErrors( - serviceId: String, - clusterName: String, + serviceId: string, + clusterName: string, ) { if (clusterName === 'empty-cluster') { return { diff --git a/plugins/newrelic/README.md b/plugins/newrelic/README.md index e14acedef2..e7cdd0df4a 100644 --- a/plugins/newrelic/README.md +++ b/plugins/newrelic/README.md @@ -15,8 +15,7 @@ proxy: '/newrelic/apm/api': target: https://api.newrelic.com/v2 headers: - X-Api-Key: - $env: NEW_RELIC_REST_API_KEY + X-Api-Key: ${NEW_RELIC_REST_API_KEY} ``` In your production deployment of Backstage, you would also need to ensure that diff --git a/plugins/rollbar-backend/README.md b/plugins/rollbar-backend/README.md index 6cf3b55948..2dcf6495dd 100644 --- a/plugins/rollbar-backend/README.md +++ b/plugins/rollbar-backend/README.md @@ -8,8 +8,7 @@ The following values are read from the configuration file. ```yaml rollbar: - accountToken: - $env: ROLLBAR_ACCOUNT_TOKEN + accountToken: ${ROLLBAR_ACCOUNT_TOKEN} ``` _NOTE: The `ROLLBAR_ACCOUNT_TOKEN` environment variable must be set to a read diff --git a/plugins/rollbar/README.md b/plugins/rollbar/README.md index 6337d45ffe..993b93d3a0 100644 --- a/plugins/rollbar/README.md +++ b/plugins/rollbar/README.md @@ -45,8 +45,7 @@ const ServiceEntityPage = ({ entity }: { entity: Entity }) => ( rollbar: organization: organization-name # used by rollbar-backend - accountToken: - $env: ROLLBAR_ACCOUNT_TOKEN + accountToken: ${ROLLBAR_ACCOUNT_TOKEN} ``` 6. Annotate entities with the rollbar project slug diff --git a/plugins/scaffolder-backend/sample-templates/v1beta2-demo/template.yaml b/plugins/scaffolder-backend/sample-templates/v1beta2-demo/template.yaml index e90b43b7f3..1a01f7998c 100644 --- a/plugins/scaffolder-backend/sample-templates/v1beta2-demo/template.yaml +++ b/plugins/scaffolder-backend/sample-templates/v1beta2-demo/template.yaml @@ -50,6 +50,7 @@ spec: values: name: '{{ parameters.name }}' owner: '{{ parameters.owner }}' + destination: '{{ parseRepoUrl parameters.repoUrl }}' - id: fetch-docs name: Fetch Docs diff --git a/plugins/scaffolder-backend/sample-templates/v1beta2-demo/template/catalog-info.yaml b/plugins/scaffolder-backend/sample-templates/v1beta2-demo/template/catalog-info.yaml index 875664d2a8..6519d68402 100644 --- a/plugins/scaffolder-backend/sample-templates/v1beta2-demo/template/catalog-info.yaml +++ b/plugins/scaffolder-backend/sample-templates/v1beta2-demo/template/catalog-info.yaml @@ -2,6 +2,7 @@ apiVersion: backstage.io/v1alpha1 kind: Component metadata: name: {{cookiecutter.name | jsonify}} + github.com/project-slug: {{cookiecutter.destination.owner + "/" + cookiecutter.destination.repo}} spec: type: website lifecycle: experimental diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/util.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/util.ts index 07b3823e7b..acf1351004 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/util.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/util.ts @@ -30,8 +30,14 @@ export const getRepoSourceDirectory = ( } return workspacePath; }; +export type RepoSpec = { + repo: string; + host: string; + owner: string; + organization?: string; +}; -export const parseRepoUrl = (repoUrl: string) => { +export const parseRepoUrl = (repoUrl: string): RepoSpec => { let parsed; try { parsed = new URL(`https://${repoUrl}`); @@ -55,7 +61,7 @@ export const parseRepoUrl = (repoUrl: string) => { ); } - const organization = parsed.searchParams.get('organization'); + const organization = parsed.searchParams.get('organization') ?? undefined; return { host, owner, repo, organization }; }; diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.test.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.test.ts index b598b91036..edef02064f 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.test.ts @@ -24,6 +24,7 @@ import { ConfigReader, JsonObject } from '@backstage/config'; import { StorageTaskBroker } from './StorageTaskBroker'; import { DatabaseTaskStore } from './DatabaseTaskStore'; import { createTemplateAction, TemplateActionRegistry } from '../actions'; +import { RepoSpec } from '../actions/builtin/publish/util'; async function createStore(): Promise { const manager = SingleConnectionDatabaseManager.fromConfig( @@ -41,20 +42,24 @@ async function createStore(): Promise { describe('TaskWorker', () => { let storage: DatabaseTaskStore; + let actionRegistry = new TemplateActionRegistry(); beforeAll(async () => { storage = await createStore(); }); - const logger = getVoidLogger(); - const actionRegistry = new TemplateActionRegistry(); - actionRegistry.register({ - id: 'test-action', - handler: async ctx => { - ctx.output('testOutput', 'winning'); - }, + beforeEach(() => { + actionRegistry = new TemplateActionRegistry(); + actionRegistry.register({ + id: 'test-action', + handler: async ctx => { + ctx.output('testOutput', 'winning'); + }, + }); }); + const logger = getVoidLogger(); + it('should fail when action does not exist', async () => { const broker = new StorageTaskBroker(storage, logger); const taskWorker = new TaskWorker({ @@ -166,4 +171,194 @@ describe('TaskWorker', () => { const event = events.find(e => e.type === 'completion'); expect((event?.body?.output as JsonObject).result).toBe('winning'); }); + + it('should parse strings as objects if possible', async () => { + const inputAction = createTemplateAction<{ + address: { line1: string }; + list: string[]; + address2: string; + }>({ + id: 'test-input', + schema: { + input: { + type: 'object', + required: ['address'], + properties: { + address: { + title: 'address', + description: 'Enter name', + type: 'object', + properties: { + line1: { + type: 'string', + }, + }, + }, + address2: { + type: 'string', + }, + list: { + type: 'array', + items: { + type: 'string', + }, + }, + }, + }, + }, + async handler(ctx) { + if (ctx.input.list.length !== 1) { + throw new Error( + `expected list to have length "1" got ${ctx.input.list.length}`, + ); + } + if (ctx.input.address.line1 !== 'line 1') { + throw new Error( + `expected address.line1 to be "line 1" got ${ctx.input.address.line1}`, + ); + } + + if (ctx.input.address2 !== '{"not valid"}') { + throw new Error( + `expected address2 to be "{"not valid"}" got ${ctx.input.address2}`, + ); + } + ctx.output('address', ctx.input.address.line1); + }, + }); + actionRegistry.register(inputAction); + + const broker = new StorageTaskBroker(storage, logger); + const taskWorker = new TaskWorker({ + logger, + workingDirectory: os.tmpdir(), + actionRegistry, + taskBroker: broker, + }); + + const { taskId } = await broker.dispatch({ + steps: [ + { + id: 'test-input', + name: 'test-input', + action: 'test-input', + input: { + address: JSON.stringify({ line1: 'line 1' }), + list: JSON.stringify(['hey!']), + address2: '{"not valid"}', + }, + }, + ], + output: { + result: '{{ steps.test-input.output.address }}', + }, + values: {}, + }); + + const task = await broker.claim(); + await taskWorker.runOneTask(task); + + const { events } = await storage.listEvents({ taskId }); + const event = events.find(e => e.type === 'completion'); + + expect((event?.body?.output as JsonObject).result).toBe('line 1'); + }); + + // TODO(blam): Can delete this test when we make the helpers a public API + it('should provide a repoUrlParse helper for the templates', async () => { + const inputAction = createTemplateAction<{ + destination: RepoSpec; + }>({ + id: 'test-input', + schema: { + input: { + type: 'object', + required: ['destination'], + properties: { + destination: { + title: 'destination', + type: 'object', + properties: { + repo: { + type: 'string', + }, + host: { + type: 'string', + }, + owner: { + type: 'string', + }, + organization: { + type: 'string', + }, + }, + }, + }, + }, + }, + async handler(ctx) { + ctx.output('host', ctx.input.destination.host); + ctx.output('repo', ctx.input.destination.repo); + ctx.output('owner', ctx.input.destination.owner); + + if (ctx.input.destination.host !== 'github.com') { + throw new Error( + `expected host to be "github.com" got ${ctx.input.destination.host}`, + ); + } + + if (ctx.input.destination.repo !== 'repo') { + throw new Error( + `expected repo to be "repo" got ${ctx.input.destination.repo}`, + ); + } + + if (ctx.input.destination.owner !== 'owner') { + throw new Error( + `expected repo to be "owner" got ${ctx.input.destination.owner}`, + ); + } + }, + }); + actionRegistry.register(inputAction); + + const broker = new StorageTaskBroker(storage, logger); + const taskWorker = new TaskWorker({ + logger, + workingDirectory: os.tmpdir(), + actionRegistry, + taskBroker: broker, + }); + + const { taskId } = await broker.dispatch({ + steps: [ + { + id: 'test-input', + name: 'test-input', + action: 'test-input', + input: { + destination: '{{ parseRepoUrl parameters.repoUrl }}', + }, + }, + ], + output: { + host: '{{ steps.test-input.output.host }}', + repo: '{{ steps.test-input.output.repo }}', + owner: '{{ steps.test-input.output.owner }}', + }, + values: { + repoUrl: 'github.com?repo=repo&owner=owner', + }, + }); + + const task = await broker.claim(); + await taskWorker.runOneTask(task); + + const { events } = await storage.listEvents({ taskId }); + const event = events.find(e => e.type === 'completion'); + + expect((event?.body?.output as JsonObject).host).toBe('github.com'); + expect((event?.body?.output as JsonObject).repo).toBe('repo'); + expect((event?.body?.output as JsonObject).owner).toBe('owner'); + }); }); diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts index 3a846f79a5..14752e1988 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts @@ -23,8 +23,9 @@ import { TaskBroker, Task } from './types'; import fs from 'fs-extra'; import path from 'path'; import { TemplateActionRegistry } from '../actions/TemplateActionRegistry'; -import * as handlebars from 'handlebars'; +import * as Handlebars from 'handlebars'; import { InputError } from '@backstage/errors'; +import { parseRepoUrl } from '../actions/builtin/publish/util'; type Options = { logger: Logger; @@ -34,7 +35,20 @@ type Options = { }; export class TaskWorker { - constructor(private readonly options: Options) {} + private readonly handlebars: typeof Handlebars; + + constructor(private readonly options: Options) { + this.handlebars = Handlebars.create(); + + // TODO(blam): this should be a public facing API but it's a little + // scary right now, so we're going to lock it off like the component API is + // in the frontend until we can work out a nice way to do it. + this.handlebars.registerHelper('parseRepoUrl', repoUrl => { + return JSON.stringify(parseRepoUrl(repoUrl)); + }); + + this.handlebars.registerHelper('json', obj => JSON.stringify(obj)); + } start() { (async () => { @@ -102,13 +116,29 @@ export class TaskWorker { step.input && JSON.parse(JSON.stringify(step.input), (_key, value) => { if (typeof value === 'string') { - return handlebars.compile(value, { + const templated = this.handlebars.compile(value, { noEscape: true, strict: true, data: false, preventIndent: true, })(templateCtx); + + // If it smells like a JSON object then give it a parse as an object and if it fails return the string + if ( + (templated.startsWith('{') && templated.endsWith('}')) || + (templated.startsWith('[') && templated.endsWith(']')) + ) { + try { + // Don't recursively JSON parse the values of this string. + // Shouldn't need to, don't want to encourage the use of returning handlebars from somewhere else + return JSON.parse(templated); + } catch { + return templated; + } + } + return templated; } + return value; }); @@ -171,7 +201,7 @@ export class TaskWorker { JSON.stringify(task.spec.output), (_key, value) => { if (typeof value === 'string') { - return handlebars.compile(value, { + return this.handlebars.compile(value, { noEscape: true, strict: true, data: false, diff --git a/plugins/sentry/README.md b/plugins/sentry/README.md index 5d280de249..52cef9cbe6 100644 --- a/plugins/sentry/README.md +++ b/plugins/sentry/README.md @@ -70,9 +70,7 @@ proxy: target: https://sentry.io/api/ allowedMethods: ['GET'] headers: - Authorization: - # Content: 'Bearer ' - $env: SENTRY_TOKEN + Authorization: Bearer ${SENTRY_TOKEN} sentry: organization: diff --git a/plugins/sonarqube/README.md b/plugins/sonarqube/README.md index f95a2c6151..269fb7c859 100644 --- a/plugins/sonarqube/README.md +++ b/plugins/sonarqube/README.md @@ -54,10 +54,9 @@ proxy: target: https://sonarcloud.io/api allowedMethods: ['GET'] headers: - Authorization: - # Content: 'Basic base64(":")' <-- note the trailing ':' - # Example: Basic bXktYXBpLWtleTo= - $env: SONARQUBE_AUTH_HEADER + Authorization: Basic ${SONARQUBE_AUTH} + # Content: 'base64(":")' <-- note the trailing ':' + # Example: bXktYXBpLWtleTo= ``` **SonarQube** @@ -70,20 +69,19 @@ proxy: target: https://your.sonarqube.instance.com/api allowedMethods: ['GET'] headers: - Authorization: - # Environmental variable: SONARQUBE_AUTH_HEADER - # Value: 'Basic base64(":")' - # Encode the ":" string using base64 encoder. - # Note the trailing colon (:) at the end of the token. - # Example environmental config: SONARQUBE_AUTH_HEADER=Basic bXktYXBpLWtleTo= - # Fetch the sonar-auth-token from https://sonarcloud.io/account/security/ - $env: SONARQUBE_AUTH_HEADER + Authorization: Basic ${SONARQUBE_AUTH} + # Environmental variable: SONARQUBE_AUTH + # Value: 'base64(":")' + # Encode the ":" string using base64 encoder. + # Note the trailing colon (:) at the end of the token. + # Example environmental config: SONARQUBE_AUTH=bXktYXBpLWtleTo= + # Fetch the sonar-auth-token from https://sonarcloud.io/account/security/ sonarQube: baseUrl: https://your.sonarqube.instance.com ``` -5. Get and provide `SONARQUBE_AUTH_HEADER` as env variable (https://sonarcloud.io/account/security or https://docs.sonarqube.org/latest/user-guide/user-token/) +5. Get and provide `SONARQUBE_AUTH` as an env variable (https://sonarcloud.io/account/security or https://docs.sonarqube.org/latest/user-guide/user-token/) 6. Run the following commands in the root folder of the project to install and compile the changes. diff --git a/plugins/splunk-on-call/README.md b/plugins/splunk-on-call/README.md index 78313d3329..5c1dda33fc 100644 --- a/plugins/splunk-on-call/README.md +++ b/plugins/splunk-on-call/README.md @@ -50,7 +50,7 @@ import { In order to be able to perform certain action (create-acknowledge-resolve an action), you need to provide a REST Endpoint. -To enable the REST Endpoint integration you can go on https://portal.victorops.com/ inside Integrations > 3rd Party Integrations > REST – Generic. +To enable the REST Endpoint integration you can go on https://portal.victorops.com/ inside Integrations > 3rd Party Integrations > REST – Generic. You can now copy the URL to notify: `/$routing_key` In `app-config.yaml`: @@ -69,10 +69,8 @@ proxy: '/splunk-on-call': target: https://api.victorops.com/api-public headers: - X-VO-Api-Id: - $env: SPLUNK_ON_CALL_API_ID - X-VO-Api-Key: - $env: SPLUNK_ON_CALL_API_KEY + X-VO-Api-Id: ${SPLUNK_ON_CALL_API_ID} + X-VO-Api-Key: ${SPLUNK_ON_CALL_API_KEY} ``` In addition, to make certain API calls (trigger-resolve-acknowledge an incident) you need to add the `PATCH` method to the backend `cors` methods list: `[GET, POST, PUT, DELETE, PATCH]`. diff --git a/plugins/techdocs/src/home/components/TechDocsHome.test.tsx b/plugins/techdocs/src/home/components/TechDocsHome.test.tsx index 9783c98f27..8067767d39 100644 --- a/plugins/techdocs/src/home/components/TechDocsHome.test.tsx +++ b/plugins/techdocs/src/home/components/TechDocsHome.test.tsx @@ -14,7 +14,13 @@ * limitations under the License. */ -import { ApiProvider, ApiRegistry } from '@backstage/core'; +import { + ApiProvider, + ApiRegistry, + ConfigApi, + configApiRef, + ConfigReader, +} from '@backstage/core'; import { CatalogApi, catalogApiRef } from '@backstage/plugin-catalog-react'; import { renderInTestApp } from '@backstage/test-utils'; import { screen } from '@testing-library/react'; @@ -26,7 +32,16 @@ describe('TechDocs Home', () => { getEntities: async () => ({ items: [] }), }; - const apiRegistry = ApiRegistry.with(catalogApiRef, catalogApi); + const configApi: ConfigApi = new ConfigReader({ + organization: { + name: 'My Company', + }, + }); + + const apiRegistry = ApiRegistry.with(catalogApiRef, catalogApi).with( + configApiRef, + configApi, + ); it('should render a TechDocs home page', async () => { await renderInTestApp( @@ -38,7 +53,7 @@ describe('TechDocs Home', () => { // Header expect(await screen.findByText('Documentation')).toBeInTheDocument(); expect( - await screen.findByText(/Documentation available in Backstage/i), + await screen.findByText(/Documentation available in My Company/i), ).toBeInTheDocument(); // Explore Content diff --git a/plugins/techdocs/src/home/components/TechDocsHome.tsx b/plugins/techdocs/src/home/components/TechDocsHome.tsx index 916a46ab7a..5e9e25e7a0 100644 --- a/plugins/techdocs/src/home/components/TechDocsHome.tsx +++ b/plugins/techdocs/src/home/components/TechDocsHome.tsx @@ -21,6 +21,8 @@ import { catalogApiRef, CatalogApi } from '@backstage/plugin-catalog-react'; import { Entity } from '@backstage/catalog-model'; import { CodeSnippet, + ConfigApi, + configApiRef, Content, Header, HeaderTabs, @@ -36,6 +38,7 @@ import { OwnedContent } from './OwnedContent'; export const TechDocsHome = () => { const [selectedTab, setSelectedTab] = useState(0); const catalogApi: CatalogApi = useApi(catalogApiRef); + const configApi: ConfigApi = useApi(configApiRef); const tabs = [{ label: 'Overview' }, { label: 'Owned Documents' }]; @@ -46,13 +49,14 @@ export const TechDocsHome = () => { }); }); + const generatedSubtitle = `Documentation available in ${ + configApi.getOptionalString('organization.name') ?? 'Backstage' + }`; + if (loading) { return ( -
+
@@ -63,10 +67,7 @@ export const TechDocsHome = () => { if (error) { return ( -
+
{ return ( -
+
setSelectedTab(index)} diff --git a/plugins/user-settings/src/components/AuthProviders/EmptyProviders.tsx b/plugins/user-settings/src/components/AuthProviders/EmptyProviders.tsx index 653f449e92..55c83eff3e 100644 --- a/plugins/user-settings/src/components/AuthProviders/EmptyProviders.tsx +++ b/plugins/user-settings/src/components/AuthProviders/EmptyProviders.tsx @@ -22,10 +22,8 @@ const EXAMPLE = `auth: providers: google: development: - clientId: - $env: AUTH_GOOGLE_CLIENT_ID - clientSecret: - $env: AUTH_GOOGLE_CLIENT_SECRET + clientId: \${AUTH_GOOGLE_CLIENT_ID} + clientSecret: \${AUTH_GOOGLE_CLIENT_SECRET} `; export const EmptyProviders = () => ( diff --git a/yarn.lock b/yarn.lock index 6f7e171b2c..1f76456d00 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4222,9 +4222,9 @@ react-lifecycles-compat "^3.0.4" "@rjsf/core@^2.4.0": - version "2.4.0" - resolved "https://registry.npmjs.org/@rjsf/core/-/core-2.4.0.tgz#c50bcff0d8178948ce08123177399d8816d51929" - integrity sha512-8zlydBkGldOxGXFEwNGFa1gzTxpcxaYn7ofegcu8XHJ7IKMCfpnU3ABg+H3eml1KZCX3FODmj1tHFJKuTmfynw== + version "2.5.1" + resolved "https://registry.npmjs.org/@rjsf/core/-/core-2.5.1.tgz#95a842d22bab5f83929662fcd73739108e9f5cbb" + integrity sha512-km8NYScXNONaL5BiSLS6wyDj49pOLZtn0iXg7Zxlm921uuf3o2AAX5SuZS5kB4Zj2zlrVMrXESexfX6bxdDYHw== dependencies: "@babel/runtime-corejs2" "^7.8.7" "@types/json-schema" "^7.0.4" @@ -4239,9 +4239,9 @@ shortid "^2.2.14" "@rjsf/material-ui@^2.4.0": - version "2.4.1" - resolved "https://registry.npmjs.org/@rjsf/material-ui/-/material-ui-2.4.1.tgz#b0dedff8877b114147e298966ca3faba895a7a62" - integrity sha512-pZaWL5Dw+km8S0hFLIK1lRHaeNtheMxTF2mZrWhf6HlGWCTGkQJnXta2UgshJN/nKtZPgO1L4FKz42Eun9nnhg== + version "2.5.1" + resolved "https://registry.npmjs.org/@rjsf/material-ui/-/material-ui-2.5.1.tgz#b93ac9f1f4a909e2aae729616859c2d72557e53e" + integrity sha512-ooKxQJO12+i1xCGtknMZDxWSbVlSEgQ5U1I7lJ+uI/MvwAg3Rz9wb2cTEF3ErBczT7EEW+j1lR19rxBjFd78MA== "@roadiehq/backstage-plugin-buildkite@^1.0.0": version "1.0.0" @@ -23398,9 +23398,9 @@ semver@7.0.0: integrity sha512-+GB6zVA9LWh6zovYQLALHwv5rb2PHGlJi3lfiqIHxR0uuwCgefcOJc59v9fv1w8GbStwxuuqqAjI9NMAOOgq1A== semver@7.x, semver@^7.0.0, semver@^7.1.1, semver@^7.1.3, semver@^7.2.1, semver@^7.3.2, semver@^7.3.4: - version "7.3.4" - resolved "https://registry.yarnpkg.com/semver/-/semver-7.3.4.tgz#27aaa7d2e4ca76452f98d3add093a72c943edc97" - integrity sha512-tCfb2WLjqFAtXn4KEdxIhalnRtoKFN7nAwj0B3ZXCbQloV2tq5eDbcTmT68JJD3nRJq24/XgxtQKFIpQdtvmVw== + version "7.3.5" + resolved "https://registry.npmjs.org/semver/-/semver-7.3.5.tgz#0b621c879348d8998e4b0e4be94b3f12e6018ef7" + integrity sha512-PoeGJYh8HK4BTO/a9Tf6ZG3veo/A7ZVsYrSA6J8ny9nb3B1VrpkuN+z9OE5wfE5p6H4LchYZsegiQgbJD94ZFQ== dependencies: lru-cache "^6.0.0"