Merge branch 'master' into k8s-backend-skipTLSVerify-configurable

Signed-off-by: Travis Truman <trumant@gmail.com>
This commit is contained in:
Travis Truman
2021-04-15 09:30:05 -04:00
73 changed files with 1150 additions and 727 deletions
+47
View File
@@ -0,0 +1,47 @@
---
'@backstage/techdocs-common': minor
---
Move the sanity checks of the publisher configurations to a dedicated `PublisherBase#getReadiness()` method instead of throwing an error when doing `Publisher.fromConfig(...)`.
You should include the check when your backend to get early feedback about a potential misconfiguration:
```diff
// packages/backend/src/plugins/techdocs.ts
export default async function createPlugin({
logger,
config,
discovery,
reader,
}: PluginEnvironment): Promise<Router> {
// ...
const publisher = await Publisher.fromConfig(config, {
logger,
discovery,
})
+ // checks if the publisher is working and logs the result
+ await publisher.getReadiness();
// Docker client (conditionally) used by the generators, based on techdocs.generators config.
const dockerClient = new Docker();
// ...
}
```
If you want to crash your application on invalid configurations, you can throw an `Error` to preserve the old behavior.
Please be aware that this is not the recommended for the use in a Backstage backend but might be helpful in CLI tools such as the `techdocs-cli`.
```ts
const publisher = await Publisher.fromConfig(config, {
logger,
discovery,
});
const ready = await publisher.getReadiness();
if (!ready.isAvailable) {
throw new Error('Invalid TechDocs publisher configuration');
}
```
+5
View File
@@ -0,0 +1,5 @@
---
'@backstage/catalog-model': patch
---
Add getEntitySourceLocation helper
+5
View File
@@ -0,0 +1,5 @@
---
'@backstage/plugin-kubernetes-backend': patch
---
Use `string` TypeScript type instead of `String`.
+38
View File
@@ -0,0 +1,38 @@
---
'@backstage/plugin-scaffolder-backend': patch
---
Enable the JSON parsing of the response from templated variables in the `v2beta1` syntax. Previously if template parameters json strings they were left as strings, they are now parsed as JSON objects.
Before:
```yaml
- id: test
name: test-action
action: custom:run
input:
input: '{"hello":"ben"}'
```
Now:
```yaml
- id: test
name: test-action
action: custom:run
input:
input:
hello: ben
```
Also added the `parseRepoUrl` and `json` helpers to the parameters syntax. You can now use these helpers to parse work with some `json` or `repoUrl` strings in templates.
```yaml
- id: test
name: test-action
action: cookiecutter:fetch
input:
destination: '{{ parseRepoUrl parameters.repoUrl }}'
```
Will produce a parsed version of the `repoUrl` of type `{ repo: string, owner: string, host: string }` that you can use in your actions. Specifically `cookiecutter` with `{{ cookiecutter.destination.owner }}` like the `plugins/scaffolder-backend/sample-templates/v1beta2-demo/template.yaml` example.
@@ -0,0 +1,6 @@
---
'@backstage/plugin-techdocs': patch
---
Reworked the TechDocs plugin to support using the configured company name instead of
'Backstage' in the page title.
+5
View File
@@ -0,0 +1,5 @@
---
'@backstage/catalog-client': patch
---
Make sure the `CatalogClient` escapes URL parameters correctly.
+33
View File
@@ -0,0 +1,33 @@
---
'@backstage/create-app': patch
---
Due to a change in the techdocs publishers, they don't check if they are able to reach e.g. the configured S3 bucket anymore.
This can be added again by the following change. Note that the backend process will no longer exit when it is not reachable but will only emit an error log message.
You should include the check when your backend to get early feedback about a potential misconfiguration:
```diff
// packages/backend/src/plugins/techdocs.ts
export default async function createPlugin({
logger,
config,
discovery,
reader,
}: PluginEnvironment): Promise<Router> {
// ...
const publisher = await Publisher.fromConfig(config, {
logger,
discovery,
})
+ // checks if the publisher is working and logs the result
+ await publisher.getReadiness();
// Docker client (conditionally) used by the generators, based on techdocs.generators config.
const dockerClient = new Docker();
// ...
}
```
+5
View File
@@ -0,0 +1,5 @@
---
'@backstage/cli': patch
---
Disable hot reloading in CI environments.
+3 -6
View File
@@ -1,13 +1,10 @@
app:
title: Backstage Tugboat Preview
baseUrl:
$env: TUGBOAT_DEFAULT_SERVICE_URL
baseUrl: ${TUGBOAT_DEFAULT_SERVICE_URL}
backend:
baseUrl:
$env: TUGBOAT_DEFAULT_SERVICE_URL
baseUrl: ${TUGBOAT_DEFAULT_SERVICE_URL}
cors:
origin:
$env: TUGBOAT_DEFAULT_SERVICE_URL
origin: ${TUGBOAT_DEFAULT_SERVICE_URL}
methods: [GET, POST, PUT, DELETE]
credentials: true
+56 -112
View File
@@ -40,47 +40,40 @@ proxy:
'/circleci/api':
target: https://circleci.com/api/v1.1
headers:
Circle-Token:
$env: CIRCLECI_AUTH_TOKEN
Circle-Token: ${CIRCLECI_AUTH_TOKEN}
'/jenkins/api':
target: http://localhost:8080
headers:
Authorization:
$env: JENKINS_BASIC_AUTH_HEADER
Authorization: ${JENKINS_BASIC_AUTH_HEADER}
'/travisci/api':
target: https://api.travis-ci.com
changeOrigin: true
headers:
Authorization:
$env: TRAVISCI_AUTH_TOKEN
Authorization: ${TRAVISCI_AUTH_TOKEN}
travis-api-version: '3'
'/newrelic/apm/api':
target: https://api.newrelic.com/v2
headers:
X-Api-Key:
$env: NEW_RELIC_REST_API_KEY
X-Api-Key: ${NEW_RELIC_REST_API_KEY}
'/pagerduty':
target: https://api.pagerduty.com
headers:
Authorization:
$env: PAGERDUTY_TOKEN
Authorization: ${PAGERDUTY_TOKEN}
'/buildkite/api':
target: https://api.buildkite.com/v2/
headers:
Authorization:
$env: BUILDKITE_TOKEN
Authorization: ${BUILDKITE_TOKEN}
'/sentry/api':
target: https://sentry.io/api/
allowedMethods: ['GET']
headers:
Authorization:
$env: SENTRY_TOKEN
Authorization: ${SENTRY_TOKEN}
organization:
name: My Company
@@ -124,36 +117,28 @@ kafka:
integrations:
github:
- host: github.com
token:
$env: GITHUB_TOKEN
token: ${GITHUB_TOKEN}
### Example for how to add your GitHub Enterprise instance using the API:
# - host: ghe.example.net
# apiBaseUrl: https://ghe.example.net/api/v3
# token:
# $env: GHE_TOKEN
# token: ${GHE_TOKEN}
### Example for how to add your GitHub Enterprise instance using raw HTTP fetches (token is optional):
# - host: ghe.example.net
# rawBaseUrl: https://ghe.example.net/raw
# token:
# $env: GHE_TOKEN
# token: ${GHE_TOKEN}
gitlab:
- host: gitlab.com
token:
$env: GITLAB_TOKEN
token: ${GITLAB_TOKEN}
bitbucket:
- host: bitbucket.org
username:
$env: BITBUCKET_USERNAME
appPassword:
$env: BITBUCKET_APP_PASSWORD
username: ${BITBUCKET_USERNAME}
appPassword: ${BITBUCKET_APP_PASSWORD}
azure:
- host: dev.azure.com
token:
$env: AZURE_TOKEN
token: ${AZURE_TOKEN}
# googleGcs:
# clientEmail: 'example@example.com'
# privateKey:
# $env: GCS_PRIVATE_KEY
# privateKey: ${GCS_PRIVATE_KEY}
catalog:
rules:
@@ -172,21 +157,18 @@ catalog:
githubOrg:
providers:
- target: https://github.com
token:
$env: GITHUB_TOKEN
token: ${GITHUB_TOKEN}
#### Example for how to add your GitHub Enterprise instance using the API:
# - target: https://ghe.example.net
# apiBaseUrl: https://ghe.example.net/api
# token:
# $env: GHE_TOKEN
# token: ${GHE_TOKEN}
ldapOrg:
### Example for how to add your enterprise LDAP server
# providers:
# - target: ldaps://ds.example.net
# bind:
# dn: uid=ldap-reader-user,ou=people,ou=example,dc=example,dc=net
# secret:
# $env: LDAP_SECRET
# secret: ${LDAP_SECRET}
# users:
# dn: ou=people,ou=example,dc=example,dc=net
# options:
@@ -202,12 +184,9 @@ catalog:
#providers:
# - target: https://graph.microsoft.com/v1.0
# authority: https://login.microsoftonline.com
# tenantId:
# $env: MICROSOFT_GRAPH_TENANT_ID
# clientId:
# $env: MICROSOFT_GRAPH_CLIENT_ID
# clientSecret:
# $env: MICROSOFT_GRAPH_CLIENT_SECRET_TOKEN
# tenantId: ${MICROSOFT_GRAPH_TENANT_ID}
# clientId: ${MICROSOFT_GRAPH_CLIENT_ID}
# clientSecret: ${MICROSOFT_GRAPH_CLIENT_SECRET_TOKEN}
# userFilter: accountEnabled eq true and userType eq 'member'
# groupFilter: securityEnabled eq false and mailEnabled eq true and groupTypes/any(c:c+eq+'Unified')
@@ -255,27 +234,22 @@ catalog:
scaffolder:
github:
token:
$env: GITHUB_TOKEN
token: ${GITHUB_TOKEN}
visibility: public # or 'internal' or 'private'
gitlab:
api:
baseUrl: https://gitlab.com
token:
$env: GITLAB_TOKEN
token: ${GITLAB_TOKEN}
visibility: public # or 'internal' or 'private'
azure:
baseUrl: https://dev.azure.com/{your-organization}
api:
token:
$env: AZURE_TOKEN
token: ${AZURE_TOKEN}
bitbucket:
api:
host: https://bitbucket.org
username:
$env: BITBUCKET_USERNAME
token:
$env: BITBUCKET_TOKEN
username: ${BITBUCKET_USERNAME}
token: ${BITBUCKET_TOKEN}
visibility: public # or or 'private'
auth:
@@ -286,89 +260,59 @@ auth:
providers:
google:
development:
clientId:
$env: AUTH_GOOGLE_CLIENT_ID
clientSecret:
$env: AUTH_GOOGLE_CLIENT_SECRET
clientId: ${AUTH_GOOGLE_CLIENT_ID}
clientSecret: ${AUTH_GOOGLE_CLIENT_SECRET}
github:
development:
clientId:
$env: AUTH_GITHUB_CLIENT_ID
clientSecret:
$env: AUTH_GITHUB_CLIENT_SECRET
enterpriseInstanceUrl:
$env: AUTH_GITHUB_ENTERPRISE_INSTANCE_URL
clientId: ${AUTH_GITHUB_CLIENT_ID}
clientSecret: ${AUTH_GITHUB_CLIENT_SECRET}
enterpriseInstanceUrl: ${AUTH_GITHUB_ENTERPRISE_INSTANCE_URL}
gitlab:
development:
clientId:
$env: AUTH_GITLAB_CLIENT_ID
clientSecret:
$env: AUTH_GITLAB_CLIENT_SECRET
audience:
$env: GITLAB_BASE_URL
clientId: ${AUTH_GITLAB_CLIENT_ID}
clientSecret: ${AUTH_GITLAB_CLIENT_SECRET}
audience: ${GITLAB_BASE_URL}
saml:
entryPoint: 'http://localhost:7001/'
issuer: 'passport-saml'
okta:
development:
clientId:
$env: AUTH_OKTA_CLIENT_ID
clientSecret:
$env: AUTH_OKTA_CLIENT_SECRET
audience:
$env: AUTH_OKTA_AUDIENCE
clientId: ${AUTH_OKTA_CLIENT_ID}
clientSecret: ${AUTH_OKTA_CLIENT_SECRET}
audience: ${AUTH_OKTA_AUDIENCE}
oauth2:
development:
clientId:
$env: AUTH_OAUTH2_CLIENT_ID
clientSecret:
$env: AUTH_OAUTH2_CLIENT_SECRET
authorizationUrl:
$env: AUTH_OAUTH2_AUTH_URL
tokenUrl:
$env: AUTH_OAUTH2_TOKEN_URL
clientId: ${AUTH_OAUTH2_CLIENT_ID}
clientSecret: ${AUTH_OAUTH2_CLIENT_SECRET}
authorizationUrl: ${AUTH_OAUTH2_AUTH_URL}
tokenUrl: ${AUTH_OAUTH2_TOKEN_URL}
###
# provide a list of scopes as needed for your OAuth2 Server:
#
# scope: saml-login-selector openid profile email
oidc:
development:
metadataUrl:
$env: AUTH_OIDC_METADATA_URL
clientId:
$env: AUTH_OIDC_CLIENT_ID
clientSecret:
$env: AUTH_OIDC_CLIENT_SECRET
authorizationUrl:
$env: AUTH_OIDC_AUTH_URL
tokenUrl:
$env: AUTH_OIDC_TOKEN_URL
tokenSignedResponseAlg:
$env: AUTH_OIDC_TOKEN_SIGNED_RESPONSE_ALG
metadataUrl: ${AUTH_OIDC_METADATA_URL}
clientId: ${AUTH_OIDC_CLIENT_ID}
clientSecret: ${AUTH_OIDC_CLIENT_SECRET}
authorizationUrl: ${AUTH_OIDC_AUTH_URL}
tokenUrl: ${AUTH_OIDC_TOKEN_URL}
tokenSignedResponseAlg: ${AUTH_OIDC_TOKEN_SIGNED_RESPONSE_ALG}
auth0:
development:
clientId:
$env: AUTH_AUTH0_CLIENT_ID
clientSecret:
$env: AUTH_AUTH0_CLIENT_SECRET
domain:
$env: AUTH_AUTH0_DOMAIN
clientId: ${AUTH_AUTH0_CLIENT_ID}
clientSecret: ${AUTH_AUTH0_CLIENT_SECRET}
domain: ${AUTH_AUTH0_DOMAIN}
microsoft:
development:
clientId:
$env: AUTH_MICROSOFT_CLIENT_ID
clientSecret:
$env: AUTH_MICROSOFT_CLIENT_SECRET
tenantId:
$env: AUTH_MICROSOFT_TENANT_ID
clientId: ${AUTH_MICROSOFT_CLIENT_ID}
clientSecret: ${AUTH_MICROSOFT_CLIENT_SECRET}
tenantId: ${AUTH_MICROSOFT_TENANT_ID}
onelogin:
development:
clientId:
$env: AUTH_ONELOGIN_CLIENT_ID
clientSecret:
$env: AUTH_ONELOGIN_CLIENT_SECRET
issuer:
$env: AUTH_ONELOGIN_ISSUER
clientId: ${AUTH_ONELOGIN_CLIENT_ID}
clientSecret: ${AUTH_ONELOGIN_CLIENT_SECRET}
issuer: ${AUTH_ONELOGIN_ISSUER}
costInsights:
engineerCost: 200000
products:
+21 -42
View File
@@ -127,68 +127,47 @@ appConfig:
development:
appOrigin: 'http://localhost:3000/'
secure: false
clientId:
$env: AUTH_GOOGLE_CLIENT_ID
clientSecret:
$env: AUTH_GOOGLE_CLIENT_SECRET
clientId: ${AUTH_GOOGLE_CLIENT_ID}
clientSecret: ${AUTH_GOOGLE_CLIENT_SECRET}
github:
development:
appOrigin: 'http://localhost:3000/'
secure: false
clientId:
$env: AUTH_GITHUB_CLIENT_ID
clientSecret:
$env: AUTH_GITHUB_CLIENT_SECRET
enterpriseInstanceUrl:
$env: AUTH_GITHUB_ENTERPRISE_INSTANCE_URL
clientId: ${AUTH_GITHUB_CLIENT_ID}
clientSecret: ${AUTH_GITHUB_CLIENT_SECRET}
enterpriseInstanceUrl: ${AUTH_GITHUB_ENTERPRISE_INSTANCE_URL}
gitlab:
development:
appOrigin: 'http://localhost:3000/'
secure: false
clientId:
$env: AUTH_GITLAB_CLIENT_ID
clientSecret:
$env: AUTH_GITLAB_CLIENT_SECRET
audience:
$env: GITLAB_BASE_URL
clientId: ${AUTH_GITLAB_CLIENT_ID}
clientSecret: ${AUTH_GITLAB_CLIENT_SECRET}
audience: ${GITLAB_BASE_URL}
okta:
development:
appOrigin: 'http://localhost:3000/'
secure: false
clientId:
$env: AUTH_OKTA_CLIENT_ID
clientSecret:
$env: AUTH_OKTA_CLIENT_SECRET
audience:
$env: AUTH_OKTA_AUDIENCE
clientId: ${AUTH_OKTA_CLIENT_ID}
clientSecret: ${AUTH_OKTA_CLIENT_SECRET}
audience: ${AUTH_OKTA_AUDIENCE}
oauth2:
development:
appOrigin: 'http://localhost:3000/'
secure: false
clientId:
$env: AUTH_OAUTH2_CLIENT_ID
clientSecret:
$env: AUTH_OAUTH2_CLIENT_SECRET
authorizationURL:
$env: AUTH_OAUTH2_AUTH_URL
tokenURL:
$env: AUTH_OAUTH2_TOKEN_URL
clientId: ${AUTH_OAUTH2_CLIENT_ID}
clientSecret: ${AUTH_OAUTH2_CLIENT_SECRET}
authorizationURL: ${AUTH_OAUTH2_AUTH_URL}
tokenURL: ${AUTH_OAUTH2_TOKEN_URL}
auth0:
development:
clientId:
$env: AUTH_AUTH0_CLIENT_ID
clientSecret:
$env: AUTH_AUTH0_CLIENT_SECRET
domain:
$env: AUTH_AUTH0_DOMAIN
clientId: ${AUTH_AUTH0_CLIENT_ID}
clientSecret: ${AUTH_AUTH0_CLIENT_SECRET}
domain: ${AUTH_AUTH0_DOMAIN}
microsoft:
development:
clientId:
$env: AUTH_MICROSOFT_CLIENT_ID
clientSecret:
$env: AUTH_MICROSOFT_CLIENT_SECRET
tenantId:
$env: AUTH_MICROSOFT_TENANT_ID
clientId: ${AUTH_MICROSOFT_CLIENT_ID}
clientSecret: ${AUTH_MICROSOFT_CLIENT_SECRET}
tenantId: ${AUTH_MICROSOFT_TENANT_ID}
auth:
google:
+14 -27
View File
@@ -95,40 +95,27 @@ auth:
providers:
google:
development:
clientId:
$env: AUTH_GOOGLE_CLIENT_ID
clientSecret:
$env: AUTH_GOOGLE_CLIENT_SECRET
clientId: ${AUTH_GOOGLE_CLIENT_ID}
clientSecret: ${AUTH_GOOGLE_CLIENT_SECRET}
github:
development:
clientId:
$env: AUTH_GITHUB_CLIENT_ID
clientSecret:
$env: AUTH_GITHUB_CLIENT_SECRET
enterpriseInstanceUrl:
$env: AUTH_GITHUB_ENTERPRISE_INSTANCE_URL
clientId: ${AUTH_GITHUB_CLIENT_ID}
clientSecret: ${AUTH_GITHUB_CLIENT_SECRET}
enterpriseInstanceUrl: ${AUTH_GITHUB_ENTERPRISE_INSTANCE_URL}
gitlab:
development:
clientId:
$env:
clientId: ${AUTH_GITLAB_CLIENT_ID}
oauth2:
development:
clientId:
$env: AUTH_OAUTH2_CLIENT_ID
clientSecret:
$env: AUTH_OAUTH2_CLIENT_SECRET
authorizationUrl:
$env: AUTH_OAUTH2_AUTH_URL
tokenUrl:
$env: AUTH_OAUTH2_TOKEN_URL
scope:
$env: AUTH_OAUTH2_SCOPE
clientId: ${AUTH_OAUTH2_CLIENT_ID}
clientSecret: ${AUTH_OAUTH2_CLIENT_SECRET}
authorizationUrl: ${AUTH_OAUTH2_AUTH_URL}
tokenUrl: ${AUTH_OAUTH2_TOKEN_URL}
scope: ${AUTH_OAUTH2_SCOPE}
saml:
entryPoint:
$env: AUTH_SAML_ENTRY_POINT
issuer:
$env: AUTH_SAML_ISSUER
...
entryPoint: ${AUTH_SAML_ENTRY_POINT}
issuer: ${AUTH_SAML_ISSUER}
...
```
## Implementing Your Own Auth Wrapper
+3
View File
@@ -129,6 +129,9 @@ variable.
$env: MY_SECRET
```
Note however, that it's often more convenient to use
[environment variable substitution](#environment-variable-substitution) instead.
### File Includes
This reads a string value from the entire contents of a text file. The file path
+1 -2
View File
@@ -26,8 +26,7 @@ kubernetes:
name: minikube
authProvider: 'serviceAccount'
skipTLSVerify: false
serviceAccountToken:
$env: K8S_MINIKUBE_TOKEN
serviceAccountToken: ${K8S_MINIKUBE_TOKEN}
- url: http://127.0.0.2:9999
name: aws-cluster-1
authProvider: 'aws'
@@ -156,8 +156,18 @@ The key points to note are:
- Call `emit` any number of times with the results of that process
- Finally return `true`
You should now be able to instantiate this class in your backend, and add it to
the `CatalogBuilder` using the `addProcessors` method.
You should now be able to add this class to your backend in
`packages/backend/src/plugins/catalog.ts`:
```diff
+ import { SystemXReaderProcessor } from '../path/to/class';
export default async function createPlugin(
env: PluginEnvironment,
): Promise<Router> {
const builder = new CatalogBuilder(env);
+ builder.addProcessor(new SystemXReaderProcessor(env.reader));
```
Start up the backend - it should now start reading from the previously
registered location and you'll see your entities start to appear in Backstage.
@@ -92,12 +92,14 @@ view and edit links need changing.
# Example:
metadata:
annotations:
backstage.io/source-location: github:https://github.com/my-org/my-service
backstage.io/source-location: url:https://github.com/my-org/my-service/
```
A `Location` reference that points to the source code of the entity (typically a
`Component`). Useful when catalog files do not get ingested from the source code
repository itself.
repository itself. If the URL points to a folder, it is important that it is
suffixed with a `'/'` in order for relative path resolution to work
consistently.
### jenkins.io/github-folder
@@ -189,8 +189,7 @@ public within the enterprise.
integrations:
github:
- host: github.com
token:
$env: GITHUB_TOKEN
token: ${GITHUB_TOKEN}
scaffolder:
github:
@@ -207,8 +206,7 @@ instance:
integrations:
gitlab:
- host: gitlab.com
token:
$env: GITLAB_TOKEN
token: ${GITLAB_TOKEN}
```
#### Bitbucket
@@ -221,8 +219,7 @@ following:
integrations:
bitbucket:
- host: bitbucket.org
token:
$env: BITBUCKET_TOKEN
token: ${BITBUCKET_TOKEN}
```
or
@@ -231,10 +228,8 @@ or
integrations:
bitbucket:
- host: bitbucket.org
appPassword:
$env: BITBUCKET_APP_PASSWORD
username:
$env: BITBUCKET_USERNAME
appPassword: ${BITBUCKET_APP_PASSWORD}
username: ${BITBUCKET_USERNAME}
```
#### Azure DevOps
@@ -249,8 +244,7 @@ verified.
integrations:
azure:
- host: dev.azure.com
token:
$env: AZURE_TOKEN
token: ${AZURE_TOKEN}
```
### Running the Backend
+6 -12
View File
@@ -65,22 +65,18 @@ techdocs:
# https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/loading-node-credentials-environment.html
# https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/loading-node-credentials-shared.html
credentials:
accessKeyId:
$env: TECHDOCS_AWSS3_ACCESS_KEY_ID_CREDENTIAL
secretAccessKey:
$env: TECHDOCS_AWSS3_SECRET_ACCESS_KEY_CREDENTIAL
accessKeyId: ${TECHDOCS_AWSS3_ACCESS_KEY_ID_CREDENTIAL}
secretAccessKey: ${TECHDOCS_AWSS3_SECRET_ACCESS_KEY_CREDENTIAL}
# (Optional) AWS Region of the bucket.
# If not set, AWS_REGION environment variable or aws config file will be used.
# https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/setting-region.html
region:
$env: AWS_REGION
region: ${AWS_REGION}
# (Optional) Endpoint URI to send requests to.
# If not set, the default endpoint is built from the configured region.
# https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#constructor-property
endpoint:
$env: AWS_ENDPOINT
endpoint: ${AWS_ENDPOINT}
# Required when techdocs.publisher.type is set to 'azureBlobStorage'. Skip otherwise.
@@ -91,13 +87,11 @@ techdocs:
# (Required) An account name is required to write to a storage blob container.
# https://docs.microsoft.com/en-us/rest/api/storageservices/authorize-with-shared-key
credentials:
accountName:
$env: TECHDOCS_AZURE_BLOB_STORAGE_ACCOUNT_NAME
accountName: ${TECHDOCS_AZURE_BLOB_STORAGE_ACCOUNT_NAME}
# (Optional) An account key is required to write to a storage container.
# If missing,AZURE_TENANT_ID, AZURE_CLIENT_ID, AZURE_CLIENT_SECRET environment variable will be used.
# https://docs.microsoft.com/en-us/azure/storage/common/storage-auth?toc=/azure/storage/blobs/toc.json
accountKey:
$env: TECHDOCS_AZURE_BLOB_STORAGE_ACCOUNT_KEY
accountKey: ${TECHDOCS_AZURE_BLOB_STORAGE_ACCOUNT_KEY}
# (Optional and Legacy) TechDocs makes API calls to techdocs-backend using this URL. e.g. get docs of an entity, get metadata, etc.
# You don't have to specify this anymore.
+15 -30
View File
@@ -95,8 +95,7 @@ techdocs:
type: 'googleGcs'
googleGcs:
bucketName: 'name-of-techdocs-storage-bucket'
credentials:
$env: GOOGLE_APPLICATION_CREDENTIALS
credentials: ${GOOGLE_APPLICATION_CREDENTIALS}
```
**4. That's it!**
@@ -179,13 +178,10 @@ techdocs:
type: 'awsS3'
awsS3:
bucketName: 'name-of-techdocs-storage-bucket'
region:
$env: AWS_REGION
region: ${AWS_REGION}
credentials:
accessKeyId:
$env: AWS_ACCESS_KEY_ID
secretAccessKey:
$env: AWS_SECRET_ACCESS_KEY
accessKeyId: ${AWS_ACCESS_KEY_ID}
secretAccessKey: ${AWS_SECRET_ACCESS_KEY}
```
Refer to the
@@ -202,8 +198,7 @@ techdocs:
type: 'awsS3'
awsS3:
bucketName: 'name-of-techdocs-storage-bucket'
region:
$env: AWS_REGION
region: ${AWS_REGION}
credentials:
roleArn: arn:aws:iam::123456789012:role/my-backstage-role
```
@@ -276,8 +271,7 @@ techdocs:
azureBlobStorage:
containerName: 'name-of-techdocs-storage-bucket'
credentials:
accountName:
$env: TECHDOCS_AZURE_BLOB_STORAGE_ACCOUNT_NAME
accountName: ${TECHDOCS_AZURE_BLOB_STORAGE_ACCOUNT_NAME}
```
**3b. Authentication using app-config.yaml**
@@ -297,10 +291,8 @@ techdocs:
azureBlobStorage:
containerName: 'name-of-techdocs-storage-bucket'
credentials:
accountName:
$env: TECHDOCS_AZURE_BLOB_STORAGE_ACCOUNT_NAME
accountKey:
$env: TECHDOCS_AZURE_BLOB_STORAGE_ACCOUNT_KEY
accountName: ${TECHDOCS_AZURE_BLOB_STORAGE_ACCOUNT_NAME}
accountKey: ${TECHDOCS_AZURE_BLOB_STORAGE_ACCOUNT_KEY}
```
**4. That's it!**
@@ -361,20 +353,13 @@ techdocs:
openStackSwift:
containerName: 'name-of-techdocs-storage-bucket'
credentials:
userName:
$env: OPENSTACK_SWIFT_STORAGE_USERNAME
password:
$env: OPENSTACK_SWIFT_STORAGE_PASSWORD
authUrl:
$env: OPENSTACK_SWIFT_STORAGE_AUTH_URL
keystoneAuthVersion:
$env: OPENSTACK_SWIFT_STORAGE_AUTH_VERSION
domainId:
$env: OPENSTACK_SWIFT_STORAGE_DOMAIN_ID
domainName:
$env: OPENSTACK_SWIFT_STORAGE_DOMAIN_NAME
region:
$env: OPENSTACK_SWIFT_STORAGE_REGION
userName: ${OPENSTACK_SWIFT_STORAGE_USERNAME}
password: ${OPENSTACK_SWIFT_STORAGE_PASSWORD}
authUrl: ${OPENSTACK_SWIFT_STORAGE_AUTH_URL}
keystoneAuthVersion: ${OPENSTACK_SWIFT_STORAGE_AUTH_VERSION}
domainId: ${OPENSTACK_SWIFT_STORAGE_DOMAIN_ID}
domainName: ${OPENSTACK_SWIFT_STORAGE_DOMAIN_NAME}
region: ${OPENSTACK_SWIFT_STORAGE_REGION}
```
**4. That's it!**
@@ -60,8 +60,7 @@ proxy:
'/circleci/api':
target: https://circleci.com/api/v1.1
headers:
Circle-Token:
$env: CIRCLECI_AUTH_TOKEN
Circle-Token: ${CIRCLECI_AUTH_TOKEN}
```
### Adding a plugin page to the Sidebar
@@ -24,10 +24,8 @@ Explicit credentials can be set in the following format:
```yaml
integrations:
googleGcs:
clientEmail:
$env: GCS_CLIENT_EMAIL
privateKey:
$env: GCS_PRIVATE_KEY
clientEmail: ${GCS_CLIENT_EMAIL}
privateKey: ${GCS_PRIVATE_KEY}
```
Then make sure the environment variables `GCS_CLIENT_EMAIL` and
+3 -2
View File
@@ -40,8 +40,9 @@ proxy:
'/larger-example/v1':
target: http://larger.example.com:8080/svc.v1
headers:
Authorization:
$env: EXAMPLE_AUTH_HEADER
Authorization: ${EXAMPLE_AUTH_HEADER}
# ...or interpolating a value into part of a string,
# Authorization: Bearer ${EXAMPLE_AUTH_TOKEN}
```
Each key under the proxy configuration entry is a route to match, below the
+14 -27
View File
@@ -79,13 +79,10 @@ auth:
providers:
github:
development:
clientId:
$env: AUTH_GITHUB_CLIENT_ID
clientSecret:
$env: AUTH_GITHUB_CLIENT_SECRET
## uncomment the following two lines if using enterprise
# enterpriseInstanceUrl:
# $env: AUTH_GITHUB_ENTERPRISE_INSTANCE_URL
clientId: ${AUTH_GITHUB_CLIENT_ID}
clientSecret: ${AUTH_GITHUB_CLIENT_SECRET}
## uncomment the following line if using enterprise
# enterpriseInstanceUrl: ${AUTH_GITHUB_ENTERPRISE_INSTANCE_URL}
```
### 2. Generate a GitHub client ID and secret
@@ -122,10 +119,8 @@ auth:
providers:
gitlab:
development:
clientId:
$env: AUTH_GITLAB_CLIENT_ID
clientSecret:
$env: AUTH_GITLAB_CLIENT_SECRET
clientId: ${AUTH_GITLAB_CLIENT_ID}
clientSecret: ${AUTH_GITLAB_CLIENT_SECRET}
audience: https://gitlab.com # Or your self-hosted GitLab instance URL
```
@@ -172,10 +167,8 @@ auth:
providers:
google:
development:
clientId:
$env: AUTH_GOOGLE_CLIENT_ID
clientSecret:
$env: AUTH_GOOGLE_CLIENT_SECRET
clientId: ${AUTH_GOOGLE_CLIENT_ID}
clientSecret: ${AUTH_GOOGLE_CLIENT_SECRET}
```
### 2. Generate Google Credentials in Google Cloud console
@@ -216,12 +209,9 @@ auth:
providers:
microsoft:
development:
clientId:
$env: AUTH_MICROSOFT_CLIENT_ID
clientSecret:
$env: AUTH_MICROSOFT_CLIENT_SECRET
tenantId:
$env: AUTH_MICROSOFT_TENANT_ID
clientId: ${AUTH_MICROSOFT_CLIENT_ID}
clientSecret: ${AUTH_MICROSOFT_CLIENT_SECRET}
tenantId: ${AUTH_MICROSOFT_TENANT_ID}
```
### 2. Create a Microsoft App Registration in Microsoft Portal
@@ -264,12 +254,9 @@ auth:
providers:
auth0:
development:
clientId:
$env: AUTH_AUTH0_CLIENT_ID
clientSecret:
$env: AUTH_AUTH0_CLIENT_SECRET
domain:
$env: AUTH_AUTH0_DOMAIN_ID
clientId: ${AUTH_AUTH0_CLIENT_ID}
clientSecret: ${AUTH_AUTH0_CLIENT_SECRET}
domain: ${AUTH_AUTH0_DOMAIN_ID}
```
### 2. Create an Auth0 application in the Auth0 management console
+7 -11
View File
@@ -38,14 +38,10 @@ backend:
+ # config options: https://node-postgres.com/api/client
+ client: pg
+ connection:
+ host:
+ $env: POSTGRES_HOST
+ port:
+ $env: POSTGRES_PORT
+ user:
+ $env: POSTGRES_USER
+ password:
+ $env: POSTGRES_PASSWORD
+ host: ${POSTGRES_HOST}
+ port: ${POSTGRES_PORT}
+ user: ${POSTGRES_USER}
+ password: ${POSTGRES_PASSWORD}
+ # https://node-postgres.com/features/ssl
+ #ssl: require # see https://www.postgresql.org/docs/current/libpq-ssl.html Table 33.1. SSL Mode Descriptions (e.g. require)
+ #ca: # if you have a CA file and want to verify it you can uncomment this section
@@ -53,9 +49,9 @@ backend:
```
If you have a `app-config.local.yaml` for local development, a similar update
If you have an `app-config.local.yaml` for local development, a similar update
should be made there. You can set the `POSTGRES_` environment variables prior to
launching Backstage, or remove the $env keys and simply set values directly for
development.
launching Backstage, or remove the `${...}` values and simply set actual values
directly for development.
The Backstage App is now ready to start up with a PostgreSQL backing database.
+1 -1
View File
@@ -15,7 +15,7 @@
"verify:sidebars": "node ./scripts/verify-sidebars"
},
"devDependencies": {
"@spotify/prettier-config": "^9.0.0",
"@spotify/prettier-config": "^10.0.0",
"docusaurus": "^2.0.0-alpha.70",
"js-yaml": "^4.0.0",
"prettier": "^2.2.1"
+4 -4
View File
@@ -909,10 +909,10 @@
resolved "https://registry.npmjs.org/@sindresorhus/is/-/is-0.7.0.tgz#9a06f4f137ee84d7df0460c1fdb1135ffa6c50fd"
integrity sha512-ONhaKPIufzzrlNbqtWFFd+jlnemX6lJAgq9ZeiZtS7I1PIf/la7CW4m83rTXRnVnsMbW2k56pGYu7AUFJD9Pow==
"@spotify/prettier-config@^9.0.0":
version "9.0.0"
resolved "https://registry.yarnpkg.com/@spotify/prettier-config/-/prettier-config-9.0.0.tgz#7b562d56573c6fc0094446fbc92b22bc318945dc"
integrity sha512-In1q0tIiqTYKAGe3KOHDcFDdZRFISyQeSeipeTHGfki23ebHRZcjxvqj5SSdBkw65D4VpSREMi0s9i5iJiMcTw==
"@spotify/prettier-config@^10.0.0":
version "10.0.0"
resolved "https://registry.yarnpkg.com/@spotify/prettier-config/-/prettier-config-10.0.0.tgz#fa076d98d2e7e6c53dd3d86a696307a7010bd056"
integrity sha512-VYOdo8P7lIScAkl02nB9KpUAuOYMManryBIBuKJkAw5D3aVtLobfmdIKvdV6MqEmGMEQPbn7w/UpnjJYhUH+IA==
"@types/cheerio@^0.22.8":
version "0.22.23"
+3
View File
@@ -48,6 +48,9 @@ export default async function createPlugin({
discovery,
});
// checks if the publisher is working and logs the result
await publisher.getReadiness();
// Docker client (conditionally) used by the generators, based on techdocs.generators config.
const dockerClient = new Docker();
+19 -5
View File
@@ -42,10 +42,14 @@ export class CatalogClient implements CatalogApi {
}
async getLocationById(
id: String,
id: string,
options?: CatalogRequestOptions,
): Promise<Location | undefined> {
return await this.requestOptional('GET', `/locations/${id}`, options);
return await this.requestOptional(
'GET',
`/locations/${encodeURIComponent(id)}`,
options,
);
}
async getEntities(
@@ -86,7 +90,9 @@ export class CatalogClient implements CatalogApi {
const { kind, namespace = 'default', name } = compoundName;
return this.requestOptional(
'GET',
`/entities/by-name/${kind}/${namespace}/${name}`,
`/entities/by-name/${encodeURIComponent(kind)}/${encodeURIComponent(
namespace,
)}/${encodeURIComponent(name)}`,
options,
);
}
@@ -171,14 +177,22 @@ export class CatalogClient implements CatalogApi {
id: string,
options?: CatalogRequestOptions,
): Promise<void> {
await this.requestIgnored('DELETE', `/locations/${id}`, options);
await this.requestIgnored(
'DELETE',
`/locations/${encodeURIComponent(id)}`,
options,
);
}
async removeEntityByUid(
uid: string,
options?: CatalogRequestOptions,
): Promise<void> {
await this.requestIgnored('DELETE', `/entities/by-uid/${uid}`, options);
await this.requestIgnored(
'DELETE',
`/entities/by-uid/${encodeURIComponent(uid)}`,
options,
);
}
//
+1 -1
View File
@@ -46,7 +46,7 @@ export interface CatalogApi {
// Locations
getLocationById(
id: String,
id: string,
options?: CatalogRequestOptions,
): Promise<Location | undefined>;
getOriginLocationByEntity(
@@ -14,7 +14,11 @@
* limitations under the License.
*/
import { parseLocationReference, stringifyLocationReference } from './helpers';
import {
getEntitySourceLocation,
parseLocationReference,
stringifyLocationReference,
} from './helpers';
describe('parseLocationReference', () => {
it('works for the simple case', () => {
@@ -68,3 +72,48 @@ describe('stringifyLocationReference', () => {
).toThrow('Unable to stringify location reference, empty target');
});
});
describe('getEntitySourceLocation', () => {
it('returns the source-location', () => {
expect(
getEntitySourceLocation({
apiVersion: 'backstage.io/v1alpha1',
kind: 'Location',
metadata: {
name: 'test',
namespace: 'default',
annotations: {
'backstage.io/source-location': 'url:https://backstage.io/foo.yaml',
'backstage.io/managed-by-location': 'url:https://spotify.com',
},
},
}),
).toEqual({ target: 'https://backstage.io/foo.yaml', type: 'url' });
});
it('returns the managed-by-location', () => {
expect(
getEntitySourceLocation({
apiVersion: 'backstage.io/v1alpha1',
kind: 'Location',
metadata: {
name: 'test',
namespace: 'default',
annotations: {
'backstage.io/managed-by-location': 'url:https://spotify.com',
},
},
}),
).toEqual({ target: 'https://spotify.com', type: 'url' });
});
it('rejects missing location annotation', () => {
expect(() =>
getEntitySourceLocation({
apiVersion: 'backstage.io/v1alpha1',
kind: 'Location',
metadata: { name: 'test', namespace: 'default' },
}),
).toThrow(`Entity 'location:default/test' is missing location`);
});
});
@@ -14,6 +14,9 @@
* limitations under the License.
*/
import { Entity, stringifyEntityRef } from '../entity';
import { LOCATION_ANNOTATION, SOURCE_LOCATION_ANNOTATION } from './annotation';
/**
* Parses a string form location reference.
*
@@ -80,3 +83,26 @@ export function stringifyLocationReference(ref: {
return `${type}:${target}`;
}
/**
* Returns the source code location of the Entity, to the extent that one exists.
*
* If the returned location type is of type 'url', the target should be readable at least
* using the UrlReader from @backstage/backend-common. If it is not of type 'url', the caller
* needs to have explicit handling of each location type or signal that it is not supported.
*/
export function getEntitySourceLocation(
entity: Entity,
): { type: string; target: string } {
const locationRef =
entity.metadata?.annotations?.[SOURCE_LOCATION_ANNOTATION] ??
entity.metadata?.annotations?.[LOCATION_ANNOTATION];
if (!locationRef) {
throw new Error(
`Entity '${stringifyEntityRef(entity)}' is missing location`,
);
}
return parseLocationReference(locationRef);
}
+5 -1
View File
@@ -19,7 +19,11 @@ export {
ORIGIN_LOCATION_ANNOTATION,
SOURCE_LOCATION_ANNOTATION,
} from './annotation';
export { parseLocationReference, stringifyLocationReference } from './helpers';
export {
parseLocationReference,
stringifyLocationReference,
getEntitySourceLocation,
} from './helpers';
export type { Location, LocationSpec } from './types';
export {
analyzeLocationSchema,
+1 -1
View File
@@ -43,7 +43,7 @@ export async function serveBundle(options: ServeOptions) {
const compiler = webpack(config);
const server = new WebpackDevServer(compiler, {
hot: true,
hot: !process.env.CI,
contentBase: paths.targetPublic,
contentBasePublicPath: config.output?.publicPath,
publicPath: config.output?.publicPath,
@@ -53,38 +53,6 @@ export const Apis = () => (
</Page>
);
export const Grpc = () => (
<Page themeId="grpc">
<Header title="Grpc catalogue" type="tool">
{labels}
</Header>
</Page>
);
export const AsyncApi = () => (
<Page themeId="asyncapi">
<Header title="Async API catalogue" type="tool">
{labels}
</Header>
</Page>
);
export const Graphql = () => (
<Page themeId="graphql">
<Header title="GraphQL API catalogue" type="tool">
{labels}
</Header>
</Page>
);
export const OpenApi = () => (
<Page themeId="openapi">
<Header title="OpenAPI catalogue" type="tool">
{labels}
</Header>
</Page>
);
export const Tool = () => (
<Page themeId="tool">
<Header title="Stand-alone tool" type="tool">
@@ -25,14 +25,10 @@ backend:
database:
client: pg
connection:
host:
$env: POSTGRES_HOST
port:
$env: POSTGRES_PORT
user:
$env: POSTGRES_USER
password:
$env: POSTGRES_PASSWORD
host: ${POSTGRES_HOST}
port: ${POSTGRES_PORT}
user: ${POSTGRES_USER}
password: ${POSTGRES_PASSWORD}
# https://node-postgres.com/features/ssl
#ssl: require # see https://www.postgresql.org/docs/current/libpq-ssl.html Table 33.1. SSL Mode Descriptions (e.g. require)
#ca: # if you have a CA file and want to verify it you can uncomment this section
@@ -43,13 +39,11 @@ backend:
integrations:
github:
- host: github.com
token:
$env: GITHUB_TOKEN
token: ${GITHUB_TOKEN}
### Example for how to add your GitHub Enterprise instance using the API:
# - host: ghe.example.net
# apiBaseUrl: https://ghe.example.net/api/v3
# token:
# $env: GHE_TOKEN
# token: ${GHE_TOKEN}
proxy:
'/test':
@@ -73,8 +67,7 @@ auth:
scaffolder:
github:
token:
$env: GITHUB_TOKEN
token: ${GITHUB_TOKEN}
visibility: public # or 'internal' or 'private'
catalog:
@@ -34,6 +34,9 @@ export default async function createPlugin({
discovery,
});
// checks if the publisher is working and logs the result
await publisher.getReadiness();
// Docker client (conditionally) used by the generators, based on techdocs.generators config.
const dockerClient = new Docker();
@@ -81,10 +81,12 @@ class Bucket {
this.bucketName = bucketName;
}
getMetadata() {
return new Promise(resolve => {
resolve('');
});
async getMetadata() {
if (this.bucketName === 'errorBucket') {
throw Error('Bucket does not exist');
}
return '';
}
upload(source: string, { destination }) {
+10 -4
View File
@@ -80,10 +80,16 @@ export class S3 {
};
}
headBucket() {
return new Promise(resolve => {
resolve('');
});
headBucket({ Bucket }) {
return {
promise: async () => {
if (Bucket === 'errorBucket') {
throw new Error('Bucket does not exist');
}
return {};
},
};
}
upload({ Key }: { Key: string }) {
@@ -13,10 +13,11 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { EventEmitter } from 'events';
import fs from 'fs-extra';
import os from 'os';
import path from 'path';
import { EventEmitter } from 'events';
import { ClientError } from 'pkgcloud';
const rootDir = os.platform() === 'win32' ? 'C:\\rootDir' : '/rootDir';
@@ -37,12 +38,11 @@ class PkgCloudStorageClient {
getFile(
containerName: string,
file: string,
callback: (err: any, file: string) => any,
callback: (err: any, file: any) => any,
) {
checkFileExists(file).then(res => {
if (!res) {
callback('File does not exist', file);
throw new Error('File does not exist');
callback('File does not exist', undefined);
} else {
callback(undefined, 'success');
}
@@ -51,13 +51,12 @@ class PkgCloudStorageClient {
getContainer(
containerName: string,
callback: (err: string, container: string) => any,
callback: (err: ClientError, container: any) => any,
) {
if (containerName !== 'mock') {
callback('Container does not exist', containerName);
throw new Error('Container does not exist');
callback(new Error('Container does not exist'), undefined);
} else {
callback('Container does not exist', 'success');
callback(undefined, 'success');
}
}
@@ -13,16 +13,16 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { getVoidLogger } from '@backstage/backend-common';
import {
Entity,
EntityName,
ENTITY_DEFAULT_NAMESPACE,
EntityName,
} from '@backstage/catalog-model';
import { ConfigReader } from '@backstage/config';
import mockFs from 'mock-fs';
import os from 'os';
import path from 'path';
import * as winston from 'winston';
import { AwsS3Publish } from './awsS3';
import { PublisherBase, TechDocsMetadata } from './types';
@@ -59,9 +59,7 @@ const getEntityRootDir = (entity: Entity) => {
return path.join(rootDir, namespace || ENTITY_DEFAULT_NAMESPACE, kind, name);
};
const logger = winston.createLogger();
jest.spyOn(logger, 'info').mockReturnValue(logger);
jest.spyOn(logger, 'error').mockReturnValue(logger);
const logger = getVoidLogger();
let publisher: PublisherBase;
@@ -87,6 +85,39 @@ beforeEach(() => {
});
describe('AwsS3Publish', () => {
describe('getReadiness', () => {
it('should validate correct config', async () => {
expect(await publisher.getReadiness()).toEqual({
isAvailable: true,
});
});
it('should reject incorrect config', async () => {
const mockConfig = new ConfigReader({
techdocs: {
requestUrl: 'http://localhost:7000',
publisher: {
type: 'awsS3',
awsS3: {
credentials: {
accessKeyId: 'accessKeyId',
secretAccessKey: 'secretAccessKey',
},
// this bucket name will throw an error
bucketName: 'errorBucket',
},
},
},
});
const errorPublisher = AwsS3Publish.fromConfig(mockConfig, logger);
expect(await errorPublisher.getReadiness()).toEqual({
isAvailable: false,
});
});
});
describe('publish', () => {
beforeEach(() => {
const entity = createMockEntity();
@@ -17,16 +17,21 @@ import { Entity, EntityName } from '@backstage/catalog-model';
import { Config } from '@backstage/config';
import aws, { Credentials } from 'aws-sdk';
import { ManagedUpload } from 'aws-sdk/clients/s3';
import { CredentialsOptions } from 'aws-sdk/lib/credentials';
import express from 'express';
import fs from 'fs-extra';
import JSON5 from 'json5';
import createLimiter from 'p-limit';
import { CredentialsOptions } from 'aws-sdk/lib/credentials';
import path from 'path';
import { Readable } from 'stream';
import { Logger } from 'winston';
import { getFileTreeRecursively, getHeadersForFileExtension } from './helpers';
import { PublisherBase, PublishRequest, TechDocsMetadata } from './types';
import {
PublisherBase,
PublishRequest,
ReadinessResponse,
TechDocsMetadata,
} from './types';
const streamToBuffer = (stream: Readable): Promise<Buffer> => {
return new Promise((resolve, reject) => {
@@ -81,30 +86,6 @@ export class AwsS3Publish implements PublisherBase {
...(endpoint && { endpoint }),
});
// Check if the defined bucket exists. Being able to connect means the configuration is good
// and the storage client will work.
storageClient.headBucket(
{
Bucket: bucketName,
},
err => {
if (err) {
logger.error(
`Could not retrieve metadata about the AWS S3 bucket ${bucketName}. ` +
'Make sure the bucket exists. Also make sure that authentication is setup either by ' +
'explicitly defining credentials and region in techdocs.publisher.awsS3 in app config or ' +
'by using environment variables. Refer to https://backstage.io/docs/features/techdocs/using-cloud-storage',
);
logger.error(`from AWS client library: ${err.message}`);
throw new Error();
} else {
logger.info(
`Successfully connected to the AWS S3 bucket ${bucketName}.`,
);
}
},
);
return new AwsS3Publish(storageClient, bucketName, logger);
}
@@ -149,6 +130,35 @@ export class AwsS3Publish implements PublisherBase {
this.logger = logger;
}
/**
* Check if the defined bucket exists. Being able to connect means the configuration is good
* and the storage client will work.
*/
async getReadiness(): Promise<ReadinessResponse> {
try {
await this.storageClient
.headBucket({ Bucket: this.bucketName })
.promise();
this.logger.info(
`Successfully connected to the AWS S3 bucket ${this.bucketName}.`,
);
return { isAvailable: true };
} catch (error) {
this.logger.error(
`Could not retrieve metadata about the AWS S3 bucket ${this.bucketName}. ` +
'Make sure the bucket exists. Also make sure that authentication is setup either by ' +
'explicitly defining credentials and region in techdocs.publisher.awsS3 in app config or ' +
'by using environment variables. Refer to https://backstage.io/docs/features/techdocs/using-cloud-storage',
);
this.logger.error(`from AWS client library`, error);
return {
isAvailable: false,
};
}
}
/**
* Upload all the files from the generated `directory` to the S3 bucket.
* Directory structure used in the bucket is - entityNamespace/entityKind/entityName/index.html
@@ -16,8 +16,8 @@
import { getVoidLogger } from '@backstage/backend-common';
import {
Entity,
EntityName,
ENTITY_DEFAULT_NAMESPACE,
EntityName,
} from '@backstage/catalog-model';
import { ConfigReader } from '@backstage/config';
import mockFs from 'mock-fs';
@@ -59,12 +59,9 @@ const getEntityRootDir = (entity: Entity) => {
return path.join(rootDir, namespace || ENTITY_DEFAULT_NAMESPACE, kind, name);
};
function createLogger() {
const logger = getVoidLogger();
jest.spyOn(logger, 'info').mockReturnValue(logger);
jest.spyOn(logger, 'error').mockReturnValue(logger);
return logger;
}
const logger = getVoidLogger();
jest.spyOn(logger, 'info').mockReturnValue(logger);
jest.spyOn(logger, 'error').mockReturnValue(logger);
let publisher: PublisherBase;
beforeEach(async () => {
@@ -85,13 +82,51 @@ beforeEach(async () => {
},
});
publisher = await AzureBlobStoragePublish.fromConfig(
mockConfig,
createLogger(),
);
publisher = AzureBlobStoragePublish.fromConfig(mockConfig, logger);
});
describe('publishing with valid credentials', () => {
describe('getReadiness', () => {
it('should validate correct config', async () => {
expect(await publisher.getReadiness()).toEqual({
isAvailable: true,
});
});
it('should reject incorrect config', async () => {
const mockConfig = new ConfigReader({
techdocs: {
requestUrl: 'http://localhost:7000',
publisher: {
type: 'azureBlobStorage',
azureBlobStorage: {
credentials: {
accountName: 'accountName',
accountKey: 'accountKey',
},
containerName: 'bad_container',
},
},
},
});
const errorPublisher = await AzureBlobStoragePublish.fromConfig(
mockConfig,
logger,
);
expect(await errorPublisher.getReadiness()).toEqual({
isAvailable: false,
});
expect(logger.error).toHaveBeenCalledWith(
expect.stringContaining(
`Could not retrieve metadata about the Azure Blob Storage container bad_container.`,
),
);
});
});
describe('publish', () => {
beforeEach(() => {
const entity = createMockEntity();
@@ -151,6 +186,60 @@ describe('publishing with valid credentials', () => {
});
mockFs.restore();
});
it('reports an error when bad account credentials', async () => {
const mockConfig = new ConfigReader({
techdocs: {
requestUrl: 'http://localhost:7000',
publisher: {
type: 'azureBlobStorage',
azureBlobStorage: {
credentials: {
accountName: 'failupload',
accountKey: 'accountKey',
},
containerName: 'containerName',
},
},
},
});
publisher = await AzureBlobStoragePublish.fromConfig(mockConfig, logger);
const entity = createMockEntity();
const entityRootDir = getEntityRootDir(entity);
mockFs({
[entityRootDir]: {
'index.html': '',
},
});
let error;
try {
await publisher.publish({
entity,
directory: entityRootDir,
});
} catch (e) {
error = e;
}
expect(error.message).toContain(
`Unable to upload file(s) to Azure Blob Storage.`,
);
expect(logger.error).toHaveBeenCalledWith(
expect.stringContaining(
`Unable to upload file(s) to Azure Blob Storage. Error: Upload failed for ${path.join(
entityRootDir,
'index.html',
)} with status code 500`,
),
);
mockFs.restore();
});
});
describe('hasDocsBeenGenerated', () => {
@@ -243,156 +332,3 @@ describe('publishing with valid credentials', () => {
});
});
});
describe('error reporting', () => {
it('reports an error when unable to read container properties', async () => {
const mockConfig = new ConfigReader({
techdocs: {
requestUrl: 'http://localhost:7000',
publisher: {
type: 'azureBlobStorage',
azureBlobStorage: {
credentials: {
accountName: 'accountName',
},
containerName: 'bad_container',
},
},
},
});
const logger = createLogger();
let error;
try {
publisher = await AzureBlobStoragePublish.fromConfig(mockConfig, logger);
} catch (e) {
error = e;
}
expect(error).toBeInstanceOf(Error);
expect(logger.error).toHaveBeenCalledWith(
expect.stringContaining(
`Could not retrieve metadata about the Azure Blob Storage container bad_container.`,
),
);
});
it('reports an error when bad account credentials', async () => {
const mockConfig = new ConfigReader({
techdocs: {
requestUrl: 'http://localhost:7000',
publisher: {
type: 'azureBlobStorage',
azureBlobStorage: {
credentials: {
accountName: 'failupload',
accountKey: 'accountKey',
},
containerName: 'containerName',
},
},
},
});
const logger = createLogger();
publisher = await AzureBlobStoragePublish.fromConfig(mockConfig, logger);
const entity = createMockEntity();
const entityRootDir = getEntityRootDir(entity);
mockFs({
[entityRootDir]: {
'index.html': '',
},
});
let error;
try {
await publisher.publish({
entity,
directory: entityRootDir,
});
} catch (e) {
error = e;
}
expect(error.message).toContain(
`Unable to upload file(s) to Azure Blob Storage.`,
);
expect(logger.error).toHaveBeenCalledWith(
expect.stringContaining(
`Unable to upload file(s) to Azure Blob Storage. Error: Upload failed for ${path.join(
entityRootDir,
'index.html',
)} with status code 500`,
),
);
mockFs.restore();
});
describe('fetchTechDocsMetadata', () => {
it('should return tech docs metadata', async () => {
const entityNameMock = createMockEntityName();
const entity = createMockEntity();
const entityRootDir = getEntityRootDir(entity);
mockFs({
[entityRootDir]: {
'techdocs_metadata.json':
'{"site_name": "backstage", "site_description": "site_content", "etag": "etag"}',
},
});
const expectedMetadata: TechDocsMetadata = {
site_name: 'backstage',
site_description: 'site_content',
etag: 'etag',
};
expect(
await publisher.fetchTechDocsMetadata(entityNameMock),
).toStrictEqual(expectedMetadata);
mockFs.restore();
});
it('should return tech docs metadata when json encoded with single quotes', async () => {
const entityNameMock = createMockEntityName();
const entity = createMockEntity();
const entityRootDir = getEntityRootDir(entity);
mockFs({
[entityRootDir]: {
'techdocs_metadata.json': `{'site_name': 'backstage', 'site_description': 'site_content', 'etag': 'etag'}`,
},
});
const expectedMetadata: TechDocsMetadata = {
site_name: 'backstage',
site_description: 'site_content',
etag: 'etag',
};
expect(
await publisher.fetchTechDocsMetadata(entityNameMock),
).toStrictEqual(expectedMetadata);
mockFs.restore();
});
it('should return an error if the techdocs_metadata.json file is not present', async () => {
const entityNameMock = createMockEntityName();
let error;
try {
await publisher.fetchTechDocsMetadata(entityNameMock);
} catch (e) {
error = e;
}
expect(error.message).toEqual(
expect.stringContaining('TechDocs metadata fetch'),
);
});
});
});
@@ -26,16 +26,18 @@ import limiterFactory from 'p-limit';
import { default as path, default as platformPath } from 'path';
import { Logger } from 'winston';
import { getFileTreeRecursively, getHeadersForFileExtension } from './helpers';
import { PublisherBase, PublishRequest, TechDocsMetadata } from './types';
import {
PublisherBase,
PublishRequest,
ReadinessResponse,
TechDocsMetadata,
} from './types';
// The number of batches that may be ongoing at the same time.
const BATCH_CONCURRENCY = 3;
export class AzureBlobStoragePublish implements PublisherBase {
static async fromConfig(
config: Config,
logger: Logger,
): Promise<PublisherBase> {
static fromConfig(config: Config, logger: Logger): PublisherBase {
let containerName = '';
try {
containerName = config.getString(
@@ -78,26 +80,6 @@ export class AzureBlobStoragePublish implements PublisherBase {
credential,
);
try {
const response = await storageClient
.getContainerClient(containerName)
.getProperties();
if (response._response.status >= 400) {
throw new Error(
`Failed to retrieve metadata from ${response._response.request.url} with status code ${response._response.status}.`,
);
}
} catch (e) {
logger.error(
`Could not retrieve metadata about the Azure Blob Storage container ${containerName}. ` +
'Make sure that the Azure project and container exist and the access key is setup correctly ' +
'techdocs.publisher.azureBlobStorage.credentials defined in app config has correct permissions. ' +
'Refer to https://backstage.io/docs/features/techdocs/using-cloud-storage',
);
throw new Error(`from Azure Blob Storage client library: ${e.message}`);
}
return new AzureBlobStoragePublish(storageClient, containerName, logger);
}
@@ -111,6 +93,37 @@ export class AzureBlobStoragePublish implements PublisherBase {
this.logger = logger;
}
async getReadiness(): Promise<ReadinessResponse> {
try {
const response = await this.storageClient
.getContainerClient(this.containerName)
.getProperties();
if (response._response.status === 200) {
return {
isAvailable: true,
};
}
if (response._response.status >= 400) {
this.logger.error(
`Failed to retrieve metadata from ${response._response.request.url} with status code ${response._response.status}.`,
);
}
} catch (e) {
this.logger.error(`from Azure Blob Storage client library: ${e.message}`);
}
this.logger.error(
`Could not retrieve metadata about the Azure Blob Storage container ${this.containerName}. ` +
'Make sure that the Azure project and container exist and the access key is setup correctly ' +
'techdocs.publisher.azureBlobStorage.credentials defined in app config has correct permissions. ' +
'Refer to https://backstage.io/docs/features/techdocs/using-cloud-storage',
);
return { isAvailable: false };
}
/**
* Upload all the files from the generated `directory` to the Azure Blob Storage container.
* Directory structure used in the container is - entityNamespace/entityKind/entityName/index.html
@@ -16,8 +16,8 @@
import { getVoidLogger } from '@backstage/backend-common';
import {
Entity,
EntityName,
ENTITY_DEFAULT_NAMESPACE,
EntityName,
} from '@backstage/catalog-model';
import { ConfigReader } from '@backstage/config';
import mockFs from 'mock-fs';
@@ -83,6 +83,35 @@ beforeEach(async () => {
});
describe('GoogleGCSPublish', () => {
describe('getReadiness', () => {
it('should validate correct config', async () => {
expect(await publisher.getReadiness()).toEqual({
isAvailable: true,
});
});
it('should reject incorrect config', async () => {
const mockConfig = new ConfigReader({
techdocs: {
requestUrl: 'http://localhost:7000',
publisher: {
type: 'googleGcs',
googleGcs: {
credentials: '{}',
bucketName: 'errorBucket',
},
},
},
});
const errorPublisher = GoogleGCSPublish.fromConfig(mockConfig, logger);
expect(await errorPublisher.getReadiness()).toEqual({
isAvailable: false,
});
});
});
describe('publish', () => {
beforeEach(() => {
const entity = createMockEntity();
@@ -26,13 +26,15 @@ import createLimiter from 'p-limit';
import path from 'path';
import { Logger } from 'winston';
import { getFileTreeRecursively, getHeadersForFileExtension } from './helpers';
import { PublisherBase, PublishRequest, TechDocsMetadata } from './types';
import {
PublisherBase,
PublishRequest,
ReadinessResponse,
TechDocsMetadata,
} from './types';
export class GoogleGCSPublish implements PublisherBase {
static async fromConfig(
config: Config,
logger: Logger,
): Promise<PublisherBase> {
static fromConfig(config: Config, logger: Logger): PublisherBase {
let bucketName = '';
try {
bucketName = config.getString('techdocs.publisher.googleGcs.bucketName');
@@ -65,21 +67,6 @@ export class GoogleGCSPublish implements PublisherBase {
}),
});
// Check if the defined bucket exists. Being able to connect means the configuration is good
// and the storage client will work.
try {
await storageClient.bucket(bucketName).getMetadata();
logger.info(`Successfully connected to the GCS bucket ${bucketName}.`);
} catch (err) {
logger.error(
`Could not retrieve metadata about the GCS bucket ${bucketName}. ` +
'Make sure the bucket exists. Also make sure that authentication is setup either by explicitly defining ' +
'techdocs.publisher.googleGcs.credentials in app config or by using environment variables. ' +
'Refer to https://backstage.io/docs/features/techdocs/using-cloud-storage',
);
throw new Error(err.message);
}
return new GoogleGCSPublish(storageClient, bucketName, logger);
}
@@ -93,6 +80,33 @@ export class GoogleGCSPublish implements PublisherBase {
this.logger = logger;
}
/**
* Check if the defined bucket exists. Being able to connect means the configuration is good
* and the storage client will work.
*/
async getReadiness(): Promise<ReadinessResponse> {
try {
await this.storageClient.bucket(this.bucketName).getMetadata();
this.logger.info(
`Successfully connected to the GCS bucket ${this.bucketName}.`,
);
return {
isAvailable: true,
};
} catch (err) {
this.logger.error(
`Could not retrieve metadata about the GCS bucket ${this.bucketName}. ` +
'Make sure the bucket exists. Also make sure that authentication is setup either by explicitly defining ' +
'techdocs.publisher.googleGcs.credentials in app config or by using environment variables. ' +
'Refer to https://backstage.io/docs/features/techdocs/using-cloud-storage',
);
this.logger.error(`from GCS client library: ${err.message}`);
return { isAvailable: false };
}
}
/**
* Upload all the files from the generated `directory` to the GCS bucket.
* Directory structure used in the bucket is - entityNamespace/entityKind/entityName/index.html
@@ -28,6 +28,7 @@ import {
PublisherBase,
PublishRequest,
PublishResponse,
ReadinessResponse,
TechDocsMetadata,
} from './types';
@@ -65,6 +66,12 @@ export class LocalPublish implements PublisherBase {
this.discovery = discovery;
}
async getReadiness(): Promise<ReadinessResponse> {
return {
isAvailable: true,
};
}
publish({ entity, directory }: PublishRequest): Promise<PublishResponse> {
const entityNamespace = entity.metadata.namespace ?? 'default';
@@ -13,16 +13,16 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { getVoidLogger } from '@backstage/backend-common';
import {
Entity,
EntityName,
ENTITY_DEFAULT_NAMESPACE,
EntityName,
} from '@backstage/catalog-model';
import { ConfigReader } from '@backstage/config';
import mockFs from 'mock-fs';
import os from 'os';
import path from 'path';
import * as winston from 'winston';
import { OpenStackSwiftPublish } from './openStackSwift';
import { PublisherBase, TechDocsMetadata } from './types';
@@ -59,9 +59,7 @@ const getEntityRootDir = (entity: Entity) => {
return path.join(rootDir, namespace || ENTITY_DEFAULT_NAMESPACE, kind, name);
};
const logger = winston.createLogger();
jest.spyOn(logger, 'info').mockReturnValue(logger);
jest.spyOn(logger, 'error').mockReturnValue(logger);
const logger = getVoidLogger();
let publisher: PublisherBase;
@@ -89,6 +87,43 @@ beforeEach(() => {
});
describe('OpenStackSwiftPublish', () => {
describe('getReadiness', () => {
it('should validate correct config', async () => {
expect(await publisher.getReadiness()).toEqual({
isAvailable: true,
});
});
it('should reject incorrect config', async () => {
const mockConfig = new ConfigReader({
techdocs: {
requestUrl: 'http://localhost:7000',
publisher: {
type: 'openStackSwift',
openStackSwift: {
credentials: {
username: 'mockuser',
password: 'verystrongpass',
},
authUrl: 'mockauthurl',
region: 'mockregion',
containerName: 'errorBucket',
},
},
},
});
const errorPublisher = OpenStackSwiftPublish.fromConfig(
mockConfig,
logger,
);
expect(await errorPublisher.getReadiness()).toEqual({
isAvailable: false,
});
});
});
describe('publish', () => {
beforeEach(() => {
const entity = createMockEntity();
@@ -15,16 +15,21 @@
*/
import { Entity, EntityName } from '@backstage/catalog-model';
import { Config } from '@backstage/config';
import { storage } from 'pkgcloud';
import express from 'express';
import fs from 'fs-extra';
import JSON5 from 'json5';
import createLimiter from 'p-limit';
import path from 'path';
import { storage } from 'pkgcloud';
import { Readable } from 'stream';
import { Logger } from 'winston';
import { getFileTreeRecursively, getHeadersForFileExtension } from './helpers';
import { PublisherBase, PublishRequest, TechDocsMetadata } from './types';
import {
PublisherBase,
PublishRequest,
ReadinessResponse,
TechDocsMetadata,
} from './types';
const streamToBuffer = (stream: Readable): Promise<Buffer> => {
return new Promise((resolve, reject) => {
@@ -70,25 +75,6 @@ export class OpenStackSwiftPublish implements PublisherBase {
region: openStackSwiftConfig.getString('region'),
});
// Check if the defined container exists. Being able to connect means the configuration is good
// and the storage client will work.
storageClient.getContainer(containerName, (err, container) => {
if (container) {
logger.info(
`Successfully connected to the OpenStack Swift container ${containerName}.`,
);
} else {
logger.error(
`Could not retrieve metadata about the OpenStack Swift container ${containerName}. ` +
'Make sure the container exists. Also make sure that authentication is setup either by ' +
'explicitly defining credentials and region in techdocs.publisher.openStackSwift in app config or ' +
'by using environment variables. Refer to https://backstage.io/docs/features/techdocs/using-cloud-storage',
);
logger.error(`from OpenStack client library: ${err.message}`);
}
});
return new OpenStackSwiftPublish(storageClient, containerName, logger);
}
@@ -102,6 +88,37 @@ export class OpenStackSwiftPublish implements PublisherBase {
this.logger = logger;
}
/*
* Check if the defined container exists. Being able to connect means the configuration is good
* and the storage client will work.
*/
getReadiness(): Promise<ReadinessResponse> {
return new Promise(resolve => {
this.storageClient.getContainer(this.containerName, (err, container) => {
if (container) {
this.logger.info(
`Successfully connected to the OpenStack Swift container ${this.containerName}.`,
);
resolve({
isAvailable: true,
});
} else {
this.logger.error(
`Could not retrieve metadata about the OpenStack Swift container ${this.containerName}. ` +
'Make sure the container exists. Also make sure that authentication is setup either by ' +
'explicitly defining credentials and region in techdocs.publisher.openStackSwift in app config or ' +
'by using environment variables. Refer to https://backstage.io/docs/features/techdocs/using-cloud-storage',
);
this.logger.error(`from OpenStack client library: ${err.message}`);
resolve({
isAvailable: false,
});
}
});
});
}
/**
* Upload all the files from the generated `directory` to the OpenStack Swift container.
* Directory structure used in the bucket is - entityNamespace/entityKind/entityName/index.html
@@ -13,16 +13,16 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Logger } from 'winston';
import { Config } from '@backstage/config';
import { PluginEndpointDiscovery } from '@backstage/backend-common';
import { PublisherType, PublisherBase } from './types';
import { LocalPublish } from './local';
import { GoogleGCSPublish } from './googleStorage';
import { PluginEndpointDiscovery } from '@backstage/backend-common';
import { Config } from '@backstage/config';
import { Logger } from 'winston';
import { AwsS3Publish } from './awsS3';
import { AzureBlobStoragePublish } from './azureBlobStorage';
import { GoogleGCSPublish } from './googleStorage';
import { LocalPublish } from './local';
import { OpenStackSwiftPublish } from './openStackSwift';
import { PublisherBase, PublisherType } from './types';
type factoryOptions = {
logger: Logger;
@@ -45,7 +45,7 @@ export class Publisher {
switch (publisherType) {
case 'googleGcs':
logger.info('Creating Google Storage Bucket publisher for TechDocs');
return await GoogleGCSPublish.fromConfig(config, logger);
return GoogleGCSPublish.fromConfig(config, logger);
case 'awsS3':
logger.info('Creating AWS S3 Bucket publisher for TechDocs');
return AwsS3Publish.fromConfig(config, logger);
@@ -37,6 +37,14 @@ export type PublishResponse = {
remoteUrl?: string;
} | void;
/**
* Result for the validation check.
*/
export type ReadinessResponse = {
/** If true, the publisher is able to interact with the backing storage. */
isAvailable: boolean;
};
/**
* Type to hold metadata found in techdocs_metadata.json and associated with each site
* @param etag ETag of the resource used to generate the site. Usually the latest commit sha of the source repository.
@@ -53,6 +61,14 @@ export type TechDocsMetadata = {
* It also provides APIs to communicate with the storage service.
*/
export interface PublisherBase {
/**
* Check if the publisher is ready. This check tries to perform certain checks to see if the
* publisher is configured correctly and can be used to publish or read documentations.
* The different implementations might e.g. use the provided service credentials to access the
* target or check if a folder/bucket is available.
*/
getReadiness(): Promise<ReadinessResponse>;
/**
* Store the generated static files onto a storage service (either local filesystem or external service).
*
+1 -1
View File
@@ -63,5 +63,5 @@ export const pageTheme: Record<string, PageTheme> = {
library: genPageTheme(colorVariants.rubyRed, shapes.wave),
other: genPageTheme(colorVariants.darkGrey, shapes.wave),
app: genPageTheme(colorVariants.toastyOrange, shapes.wave),
apis: genPageTheme(colorVariants.eveningSea, shapes.wave2),
apis: genPageTheme(colorVariants.teal, shapes.wave2),
};
+1 -2
View File
@@ -55,8 +55,7 @@ proxy:
target: 'https://api.bitrise.io/v0.1'
allowedMethods: ['GET']
headers:
Authorization:
$env: BITRISE_AUTH_TOKEN
Authorization: ${BITRISE_AUTH_TOKEN}
```
Learn on https://devcenter.bitrise.io/api/authentication how to create a new Bitrise token.
+1 -1
View File
@@ -42,7 +42,7 @@ export class CatalogClientWrapper implements CatalogApi {
}
async getLocationById(
id: String,
id: string,
options?: CatalogRequestOptions,
): Promise<Location | undefined> {
return await this.client.getLocationById(id, {
+1 -2
View File
@@ -43,8 +43,7 @@ proxy:
'/circleci/api':
target: https://circleci.com/api/v1.1
headers:
Circle-Token:
$env: CIRCLECI_AUTH_TOKEN
Circle-Token: ${CIRCLECI_AUTH_TOKEN}
```
5. Get and provide `CIRCLECI_AUTH_TOKEN` as env variable (https://circleci.com/docs/api/#add-an-api-token)
+1 -3
View File
@@ -50,9 +50,7 @@ proxy:
target: https://app.fossa.io/api
allowedMethods: ['GET']
headers:
Authorization:
# Content: 'token <your-fossa-api-token>'
$env: FOSSA_AUTH_HEADER
Authorization: token ${FOSSA_API_TOKEN}
# if you have a fossa organization, configure your id here
fossa:
+2 -4
View File
@@ -29,15 +29,13 @@ proxy:
target: 'http://localhost:8080' # your Jenkins URL
changeOrigin: true
headers:
Authorization:
$env: JENKINS_BASIC_AUTH_HEADER
Authorization: Basic ${JENKINS_BASIC_AUTH_HEADER}
```
4. Add an environment variable which contains the Jenkins credentials, (note: use an API token not your password). Here user is the name of the user created in Jenkins.
```shell
HEADER=$(echo -n user:api-token | base64)
export JENKINS_BASIC_AUTH_HEADER="Basic $HEADER"
export JENKINS_BASIC_AUTH_HEADER=$(echo -n user:api-token | base64)
```
5. Run app with `yarn start`
@@ -21,7 +21,7 @@ import { AwsIamKubernetesAuthTranslator } from './AwsIamKubernetesAuthTranslator
export class KubernetesAuthTranslatorGenerator {
static getKubernetesAuthTranslatorInstance(
authProvider: String,
authProvider: string,
): KubernetesAuthTranslator {
switch (authProvider) {
case 'google': {
@@ -34,8 +34,8 @@ const mockFetch = (mock: jest.Mock) => {
};
function generateMockResourcesAndErrors(
serviceId: String,
clusterName: String,
serviceId: string,
clusterName: string,
) {
if (clusterName === 'empty-cluster') {
return {
+1 -2
View File
@@ -15,8 +15,7 @@ proxy:
'/newrelic/apm/api':
target: https://api.newrelic.com/v2
headers:
X-Api-Key:
$env: NEW_RELIC_REST_API_KEY
X-Api-Key: ${NEW_RELIC_REST_API_KEY}
```
In your production deployment of Backstage, you would also need to ensure that
+1 -2
View File
@@ -8,8 +8,7 @@ The following values are read from the configuration file.
```yaml
rollbar:
accountToken:
$env: ROLLBAR_ACCOUNT_TOKEN
accountToken: ${ROLLBAR_ACCOUNT_TOKEN}
```
_NOTE: The `ROLLBAR_ACCOUNT_TOKEN` environment variable must be set to a read
+1 -2
View File
@@ -45,8 +45,7 @@ const ServiceEntityPage = ({ entity }: { entity: Entity }) => (
rollbar:
organization: organization-name
# used by rollbar-backend
accountToken:
$env: ROLLBAR_ACCOUNT_TOKEN
accountToken: ${ROLLBAR_ACCOUNT_TOKEN}
```
6. Annotate entities with the rollbar project slug
@@ -50,6 +50,7 @@ spec:
values:
name: '{{ parameters.name }}'
owner: '{{ parameters.owner }}'
destination: '{{ parseRepoUrl parameters.repoUrl }}'
- id: fetch-docs
name: Fetch Docs
@@ -2,6 +2,7 @@ apiVersion: backstage.io/v1alpha1
kind: Component
metadata:
name: {{cookiecutter.name | jsonify}}
github.com/project-slug: {{cookiecutter.destination.owner + "/" + cookiecutter.destination.repo}}
spec:
type: website
lifecycle: experimental
@@ -30,8 +30,14 @@ export const getRepoSourceDirectory = (
}
return workspacePath;
};
export type RepoSpec = {
repo: string;
host: string;
owner: string;
organization?: string;
};
export const parseRepoUrl = (repoUrl: string) => {
export const parseRepoUrl = (repoUrl: string): RepoSpec => {
let parsed;
try {
parsed = new URL(`https://${repoUrl}`);
@@ -55,7 +61,7 @@ export const parseRepoUrl = (repoUrl: string) => {
);
}
const organization = parsed.searchParams.get('organization');
const organization = parsed.searchParams.get('organization') ?? undefined;
return { host, owner, repo, organization };
};
@@ -24,6 +24,7 @@ import { ConfigReader, JsonObject } from '@backstage/config';
import { StorageTaskBroker } from './StorageTaskBroker';
import { DatabaseTaskStore } from './DatabaseTaskStore';
import { createTemplateAction, TemplateActionRegistry } from '../actions';
import { RepoSpec } from '../actions/builtin/publish/util';
async function createStore(): Promise<DatabaseTaskStore> {
const manager = SingleConnectionDatabaseManager.fromConfig(
@@ -41,20 +42,24 @@ async function createStore(): Promise<DatabaseTaskStore> {
describe('TaskWorker', () => {
let storage: DatabaseTaskStore;
let actionRegistry = new TemplateActionRegistry();
beforeAll(async () => {
storage = await createStore();
});
const logger = getVoidLogger();
const actionRegistry = new TemplateActionRegistry();
actionRegistry.register({
id: 'test-action',
handler: async ctx => {
ctx.output('testOutput', 'winning');
},
beforeEach(() => {
actionRegistry = new TemplateActionRegistry();
actionRegistry.register({
id: 'test-action',
handler: async ctx => {
ctx.output('testOutput', 'winning');
},
});
});
const logger = getVoidLogger();
it('should fail when action does not exist', async () => {
const broker = new StorageTaskBroker(storage, logger);
const taskWorker = new TaskWorker({
@@ -166,4 +171,194 @@ describe('TaskWorker', () => {
const event = events.find(e => e.type === 'completion');
expect((event?.body?.output as JsonObject).result).toBe('winning');
});
it('should parse strings as objects if possible', async () => {
const inputAction = createTemplateAction<{
address: { line1: string };
list: string[];
address2: string;
}>({
id: 'test-input',
schema: {
input: {
type: 'object',
required: ['address'],
properties: {
address: {
title: 'address',
description: 'Enter name',
type: 'object',
properties: {
line1: {
type: 'string',
},
},
},
address2: {
type: 'string',
},
list: {
type: 'array',
items: {
type: 'string',
},
},
},
},
},
async handler(ctx) {
if (ctx.input.list.length !== 1) {
throw new Error(
`expected list to have length "1" got ${ctx.input.list.length}`,
);
}
if (ctx.input.address.line1 !== 'line 1') {
throw new Error(
`expected address.line1 to be "line 1" got ${ctx.input.address.line1}`,
);
}
if (ctx.input.address2 !== '{"not valid"}') {
throw new Error(
`expected address2 to be "{"not valid"}" got ${ctx.input.address2}`,
);
}
ctx.output('address', ctx.input.address.line1);
},
});
actionRegistry.register(inputAction);
const broker = new StorageTaskBroker(storage, logger);
const taskWorker = new TaskWorker({
logger,
workingDirectory: os.tmpdir(),
actionRegistry,
taskBroker: broker,
});
const { taskId } = await broker.dispatch({
steps: [
{
id: 'test-input',
name: 'test-input',
action: 'test-input',
input: {
address: JSON.stringify({ line1: 'line 1' }),
list: JSON.stringify(['hey!']),
address2: '{"not valid"}',
},
},
],
output: {
result: '{{ steps.test-input.output.address }}',
},
values: {},
});
const task = await broker.claim();
await taskWorker.runOneTask(task);
const { events } = await storage.listEvents({ taskId });
const event = events.find(e => e.type === 'completion');
expect((event?.body?.output as JsonObject).result).toBe('line 1');
});
// TODO(blam): Can delete this test when we make the helpers a public API
it('should provide a repoUrlParse helper for the templates', async () => {
const inputAction = createTemplateAction<{
destination: RepoSpec;
}>({
id: 'test-input',
schema: {
input: {
type: 'object',
required: ['destination'],
properties: {
destination: {
title: 'destination',
type: 'object',
properties: {
repo: {
type: 'string',
},
host: {
type: 'string',
},
owner: {
type: 'string',
},
organization: {
type: 'string',
},
},
},
},
},
},
async handler(ctx) {
ctx.output('host', ctx.input.destination.host);
ctx.output('repo', ctx.input.destination.repo);
ctx.output('owner', ctx.input.destination.owner);
if (ctx.input.destination.host !== 'github.com') {
throw new Error(
`expected host to be "github.com" got ${ctx.input.destination.host}`,
);
}
if (ctx.input.destination.repo !== 'repo') {
throw new Error(
`expected repo to be "repo" got ${ctx.input.destination.repo}`,
);
}
if (ctx.input.destination.owner !== 'owner') {
throw new Error(
`expected repo to be "owner" got ${ctx.input.destination.owner}`,
);
}
},
});
actionRegistry.register(inputAction);
const broker = new StorageTaskBroker(storage, logger);
const taskWorker = new TaskWorker({
logger,
workingDirectory: os.tmpdir(),
actionRegistry,
taskBroker: broker,
});
const { taskId } = await broker.dispatch({
steps: [
{
id: 'test-input',
name: 'test-input',
action: 'test-input',
input: {
destination: '{{ parseRepoUrl parameters.repoUrl }}',
},
},
],
output: {
host: '{{ steps.test-input.output.host }}',
repo: '{{ steps.test-input.output.repo }}',
owner: '{{ steps.test-input.output.owner }}',
},
values: {
repoUrl: 'github.com?repo=repo&owner=owner',
},
});
const task = await broker.claim();
await taskWorker.runOneTask(task);
const { events } = await storage.listEvents({ taskId });
const event = events.find(e => e.type === 'completion');
expect((event?.body?.output as JsonObject).host).toBe('github.com');
expect((event?.body?.output as JsonObject).repo).toBe('repo');
expect((event?.body?.output as JsonObject).owner).toBe('owner');
});
});
@@ -23,8 +23,9 @@ import { TaskBroker, Task } from './types';
import fs from 'fs-extra';
import path from 'path';
import { TemplateActionRegistry } from '../actions/TemplateActionRegistry';
import * as handlebars from 'handlebars';
import * as Handlebars from 'handlebars';
import { InputError } from '@backstage/errors';
import { parseRepoUrl } from '../actions/builtin/publish/util';
type Options = {
logger: Logger;
@@ -34,7 +35,20 @@ type Options = {
};
export class TaskWorker {
constructor(private readonly options: Options) {}
private readonly handlebars: typeof Handlebars;
constructor(private readonly options: Options) {
this.handlebars = Handlebars.create();
// TODO(blam): this should be a public facing API but it's a little
// scary right now, so we're going to lock it off like the component API is
// in the frontend until we can work out a nice way to do it.
this.handlebars.registerHelper('parseRepoUrl', repoUrl => {
return JSON.stringify(parseRepoUrl(repoUrl));
});
this.handlebars.registerHelper('json', obj => JSON.stringify(obj));
}
start() {
(async () => {
@@ -102,13 +116,29 @@ export class TaskWorker {
step.input &&
JSON.parse(JSON.stringify(step.input), (_key, value) => {
if (typeof value === 'string') {
return handlebars.compile(value, {
const templated = this.handlebars.compile(value, {
noEscape: true,
strict: true,
data: false,
preventIndent: true,
})(templateCtx);
// If it smells like a JSON object then give it a parse as an object and if it fails return the string
if (
(templated.startsWith('{') && templated.endsWith('}')) ||
(templated.startsWith('[') && templated.endsWith(']'))
) {
try {
// Don't recursively JSON parse the values of this string.
// Shouldn't need to, don't want to encourage the use of returning handlebars from somewhere else
return JSON.parse(templated);
} catch {
return templated;
}
}
return templated;
}
return value;
});
@@ -171,7 +201,7 @@ export class TaskWorker {
JSON.stringify(task.spec.output),
(_key, value) => {
if (typeof value === 'string') {
return handlebars.compile(value, {
return this.handlebars.compile(value, {
noEscape: true,
strict: true,
data: false,
+1 -3
View File
@@ -70,9 +70,7 @@ proxy:
target: https://sentry.io/api/
allowedMethods: ['GET']
headers:
Authorization:
# Content: 'Bearer <your-sentry-token>'
$env: SENTRY_TOKEN
Authorization: Bearer ${SENTRY_TOKEN}
sentry:
organization: <your-organization>
+11 -13
View File
@@ -54,10 +54,9 @@ proxy:
target: https://sonarcloud.io/api
allowedMethods: ['GET']
headers:
Authorization:
# Content: 'Basic base64("<api-key>:")' <-- note the trailing ':'
# Example: Basic bXktYXBpLWtleTo=
$env: SONARQUBE_AUTH_HEADER
Authorization: Basic ${SONARQUBE_AUTH}
# Content: 'base64("<api-key>:")' <-- note the trailing ':'
# Example: bXktYXBpLWtleTo=
```
**SonarQube**
@@ -70,20 +69,19 @@ proxy:
target: https://your.sonarqube.instance.com/api
allowedMethods: ['GET']
headers:
Authorization:
# Environmental variable: SONARQUBE_AUTH_HEADER
# Value: 'Basic base64("<sonar-auth-token>:")'
# Encode the "<sonar-auth-token>:" string using base64 encoder.
# Note the trailing colon (:) at the end of the token.
# Example environmental config: SONARQUBE_AUTH_HEADER=Basic bXktYXBpLWtleTo=
# Fetch the sonar-auth-token from https://sonarcloud.io/account/security/
$env: SONARQUBE_AUTH_HEADER
Authorization: Basic ${SONARQUBE_AUTH}
# Environmental variable: SONARQUBE_AUTH
# Value: 'base64("<sonar-auth-token>:")'
# Encode the "<sonar-auth-token>:" string using base64 encoder.
# Note the trailing colon (:) at the end of the token.
# Example environmental config: SONARQUBE_AUTH=bXktYXBpLWtleTo=
# Fetch the sonar-auth-token from https://sonarcloud.io/account/security/
sonarQube:
baseUrl: https://your.sonarqube.instance.com
```
5. Get and provide `SONARQUBE_AUTH_HEADER` as env variable (https://sonarcloud.io/account/security or https://docs.sonarqube.org/latest/user-guide/user-token/)
5. Get and provide `SONARQUBE_AUTH` as an env variable (https://sonarcloud.io/account/security or https://docs.sonarqube.org/latest/user-guide/user-token/)
6. Run the following commands in the root folder of the project to install and compile the changes.
+3 -5
View File
@@ -50,7 +50,7 @@ import {
In order to be able to perform certain action (create-acknowledge-resolve an action), you need to provide a REST Endpoint.
To enable the REST Endpoint integration you can go on https://portal.victorops.com/ inside Integrations > 3rd Party Integrations > REST Generic.
To enable the REST Endpoint integration you can go on https://portal.victorops.com/ inside Integrations > 3rd Party Integrations > REST Generic.
You can now copy the URL to notify: `<SPLUNK_ON_CALL_REST_ENDPOINT>/$routing_key`
In `app-config.yaml`:
@@ -69,10 +69,8 @@ proxy:
'/splunk-on-call':
target: https://api.victorops.com/api-public
headers:
X-VO-Api-Id:
$env: SPLUNK_ON_CALL_API_ID
X-VO-Api-Key:
$env: SPLUNK_ON_CALL_API_KEY
X-VO-Api-Id: ${SPLUNK_ON_CALL_API_ID}
X-VO-Api-Key: ${SPLUNK_ON_CALL_API_KEY}
```
In addition, to make certain API calls (trigger-resolve-acknowledge an incident) you need to add the `PATCH` method to the backend `cors` methods list: `[GET, POST, PUT, DELETE, PATCH]`.
@@ -14,7 +14,13 @@
* limitations under the License.
*/
import { ApiProvider, ApiRegistry } from '@backstage/core';
import {
ApiProvider,
ApiRegistry,
ConfigApi,
configApiRef,
ConfigReader,
} from '@backstage/core';
import { CatalogApi, catalogApiRef } from '@backstage/plugin-catalog-react';
import { renderInTestApp } from '@backstage/test-utils';
import { screen } from '@testing-library/react';
@@ -26,7 +32,16 @@ describe('TechDocs Home', () => {
getEntities: async () => ({ items: [] }),
};
const apiRegistry = ApiRegistry.with(catalogApiRef, catalogApi);
const configApi: ConfigApi = new ConfigReader({
organization: {
name: 'My Company',
},
});
const apiRegistry = ApiRegistry.with(catalogApiRef, catalogApi).with(
configApiRef,
configApi,
);
it('should render a TechDocs home page', async () => {
await renderInTestApp(
@@ -38,7 +53,7 @@ describe('TechDocs Home', () => {
// Header
expect(await screen.findByText('Documentation')).toBeInTheDocument();
expect(
await screen.findByText(/Documentation available in Backstage/i),
await screen.findByText(/Documentation available in My Company/i),
).toBeInTheDocument();
// Explore Content
@@ -21,6 +21,8 @@ import { catalogApiRef, CatalogApi } from '@backstage/plugin-catalog-react';
import { Entity } from '@backstage/catalog-model';
import {
CodeSnippet,
ConfigApi,
configApiRef,
Content,
Header,
HeaderTabs,
@@ -36,6 +38,7 @@ import { OwnedContent } from './OwnedContent';
export const TechDocsHome = () => {
const [selectedTab, setSelectedTab] = useState<number>(0);
const catalogApi: CatalogApi = useApi(catalogApiRef);
const configApi: ConfigApi = useApi(configApiRef);
const tabs = [{ label: 'Overview' }, { label: 'Owned Documents' }];
@@ -46,13 +49,14 @@ export const TechDocsHome = () => {
});
});
const generatedSubtitle = `Documentation available in ${
configApi.getOptionalString('organization.name') ?? 'Backstage'
}`;
if (loading) {
return (
<Page themeId="documentation">
<Header
title="Documentation"
subtitle="Documentation available in Backstage"
/>
<Header title="Documentation" subtitle={generatedSubtitle} />
<Content>
<Progress />
</Content>
@@ -63,10 +67,7 @@ export const TechDocsHome = () => {
if (error) {
return (
<Page themeId="documentation">
<Header
title="Documentation"
subtitle="Documentation available in Backstage"
/>
<Header title="Documentation" subtitle={generatedSubtitle} />
<Content>
<WarningPanel
severity="error"
@@ -81,10 +82,7 @@ export const TechDocsHome = () => {
return (
<Page themeId="documentation">
<Header
title="Documentation"
subtitle="Documentation available in Backstage"
/>
<Header title="Documentation" subtitle={generatedSubtitle} />
<HeaderTabs
selectedIndex={selectedTab}
onChange={index => setSelectedTab(index)}
@@ -22,10 +22,8 @@ const EXAMPLE = `auth:
providers:
google:
development:
clientId:
$env: AUTH_GOOGLE_CLIENT_ID
clientSecret:
$env: AUTH_GOOGLE_CLIENT_SECRET
clientId: \${AUTH_GOOGLE_CLIENT_ID}
clientSecret: \${AUTH_GOOGLE_CLIENT_SECRET}
`;
export const EmptyProviders = () => (
+9 -9
View File
@@ -4222,9 +4222,9 @@
react-lifecycles-compat "^3.0.4"
"@rjsf/core@^2.4.0":
version "2.4.0"
resolved "https://registry.npmjs.org/@rjsf/core/-/core-2.4.0.tgz#c50bcff0d8178948ce08123177399d8816d51929"
integrity sha512-8zlydBkGldOxGXFEwNGFa1gzTxpcxaYn7ofegcu8XHJ7IKMCfpnU3ABg+H3eml1KZCX3FODmj1tHFJKuTmfynw==
version "2.5.1"
resolved "https://registry.npmjs.org/@rjsf/core/-/core-2.5.1.tgz#95a842d22bab5f83929662fcd73739108e9f5cbb"
integrity sha512-km8NYScXNONaL5BiSLS6wyDj49pOLZtn0iXg7Zxlm921uuf3o2AAX5SuZS5kB4Zj2zlrVMrXESexfX6bxdDYHw==
dependencies:
"@babel/runtime-corejs2" "^7.8.7"
"@types/json-schema" "^7.0.4"
@@ -4239,9 +4239,9 @@
shortid "^2.2.14"
"@rjsf/material-ui@^2.4.0":
version "2.4.1"
resolved "https://registry.npmjs.org/@rjsf/material-ui/-/material-ui-2.4.1.tgz#b0dedff8877b114147e298966ca3faba895a7a62"
integrity sha512-pZaWL5Dw+km8S0hFLIK1lRHaeNtheMxTF2mZrWhf6HlGWCTGkQJnXta2UgshJN/nKtZPgO1L4FKz42Eun9nnhg==
version "2.5.1"
resolved "https://registry.npmjs.org/@rjsf/material-ui/-/material-ui-2.5.1.tgz#b93ac9f1f4a909e2aae729616859c2d72557e53e"
integrity sha512-ooKxQJO12+i1xCGtknMZDxWSbVlSEgQ5U1I7lJ+uI/MvwAg3Rz9wb2cTEF3ErBczT7EEW+j1lR19rxBjFd78MA==
"@roadiehq/backstage-plugin-buildkite@^1.0.0":
version "1.0.0"
@@ -23398,9 +23398,9 @@ semver@7.0.0:
integrity sha512-+GB6zVA9LWh6zovYQLALHwv5rb2PHGlJi3lfiqIHxR0uuwCgefcOJc59v9fv1w8GbStwxuuqqAjI9NMAOOgq1A==
semver@7.x, semver@^7.0.0, semver@^7.1.1, semver@^7.1.3, semver@^7.2.1, semver@^7.3.2, semver@^7.3.4:
version "7.3.4"
resolved "https://registry.yarnpkg.com/semver/-/semver-7.3.4.tgz#27aaa7d2e4ca76452f98d3add093a72c943edc97"
integrity sha512-tCfb2WLjqFAtXn4KEdxIhalnRtoKFN7nAwj0B3ZXCbQloV2tq5eDbcTmT68JJD3nRJq24/XgxtQKFIpQdtvmVw==
version "7.3.5"
resolved "https://registry.npmjs.org/semver/-/semver-7.3.5.tgz#0b621c879348d8998e4b0e4be94b3f12e6018ef7"
integrity sha512-PoeGJYh8HK4BTO/a9Tf6ZG3veo/A7ZVsYrSA6J8ny9nb3B1VrpkuN+z9OE5wfE5p6H4LchYZsegiQgbJD94ZFQ==
dependencies:
lru-cache "^6.0.0"