paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 46 Packages Projects Releases Wiki Activity

Merge pull request #2741 from spotify/rugvip/short

Browse Source 
config-loader: add support for new short-form secret syntax
This commit is contained in:
Patrik Oldsberg
2020-10-05 13:30:04 +02:00
committed by GitHub
parent 86e5c629de eaf169c07a
commit 34eb3a13c8
19 changed files with 188 additions and 179 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.changeset/short-secrets.md
+5
View File
@@ -0,0 +1,5 @@
---
'@backstage/config-loader': minor
---
Added support for new shorthand when defining secrets, where `$env: ENV` can be used instead of `$secret: { env: ENV }` etc.
app-config.yaml
+37 -74
View File
@@ -18,23 +18,20 @@ proxy:
target: https://circleci.com/api/v1.1
headers:
Circle-Token:
$secret:
env: CIRCLECI_AUTH_TOKEN
$env: CIRCLECI_AUTH_TOKEN
'/jenkins/api':
target: http://localhost:8080
headers:
Authorization:
$secret:
env: JENKINS_BASIC_AUTH_HEADER
$env: JENKINS_BASIC_AUTH_HEADER
'/travisci/api':
target: https://api.travis-ci.com
changeOrigin: true
headers:
Authorization:
$secret:
env: TRAVISCI_AUTH_TOKEN
$env: TRAVISCI_AUTH_TOKEN
travis-api-version: 3
organization:
@@ -52,8 +49,7 @@ sentry:
rollbar:
organization: spotify
accountToken:
$secret:
env: ROLLBAR_ACCOUNT_TOKEN
$env: ROLLBAR_ACCOUNT_TOKEN
newrelic:
api:
@@ -71,38 +67,31 @@ integrations:
github:
- host: github.com
token:
$secret:
env: GITHUB_PRIVATE_TOKEN
$env: GITHUB_PRIVATE_TOKEN
### Example for how to add your GitHub Enterprise instance using the API:
# - host: ghe.example.net
# apiBaseUrl: https://ghe.example.net/api/v3
# token:
# $secret:
# env: GHE_PRIVATE_TOKEN
# $env: GHE_PRIVATE_TOKEN
### Example for how to add your GitHub Enterprise instance using raw HTTP fetches (token is optional):
# - host: ghe.example.net
# rawBaseUrl: https://ghe.example.net/raw
# token:
# $secret:
# env: GHE_PRIVATE_TOKEN
# $env: GHE_PRIVATE_TOKEN
gitlab:
- host: gitlab.com
token:
$secret:
env: GITLAB_PRIVATE_TOKEN
$env: GITLAB_PRIVATE_TOKEN
bitbucket:
- host: bitbucket.org
username:
$secret:
env: BITBUCKET_USERNAME
$env: BITBUCKET_USERNAME
appPassword:
$secret:
env: BITBUCKET_APP_PASSWORD
$env: BITBUCKET_APP_PASSWORD
azure:
- host: dev.azure.com
token:
$secret:
env: AZURE_PRIVATE_TOKEN
$env: AZURE_PRIVATE_TOKEN
catalog:
rules:
@@ -113,14 +102,12 @@ catalog:
providers:
- target: https://github.com
token:
$secret:
env: GITHUB_PRIVATE_TOKEN
$env: GITHUB_PRIVATE_TOKEN
#### Example for how to add your GitHub Enterprise instance using the API:
# - target: https://ghe.example.net
# apiBaseUrl: https://ghe.example.net/api/v3
# token:
# $secret:
# env: GHE_PRIVATE_TOKEN
# $env: GHE_PRIVATE_TOKEN
locations:
# Backstage example components
@@ -142,104 +129,80 @@ catalog:
scaffolder:
github:
token:
$secret:
env: GITHUB_ACCESS_TOKEN
$env: GITHUB_ACCESS_TOKEN
visibility: public # or 'internal' or 'private'
gitlab:
api:
baseUrl: https://gitlab.com
token:
$secret:
env: GITLAB_ACCESS_TOKEN
$env: GITLAB_ACCESS_TOKEN
azure:
baseUrl: https://dev.azure.com/{your-organization}
api:
token:
$secret:
env: AZURE_PRIVATE_TOKEN
$env: AZURE_PRIVATE_TOKEN
auth:
providers:
google:
development:
clientId:
$secret:
env: AUTH_GOOGLE_CLIENT_ID
$env: AUTH_GOOGLE_CLIENT_ID
clientSecret:
$secret:
env: AUTH_GOOGLE_CLIENT_SECRET
$env: AUTH_GOOGLE_CLIENT_SECRET
github:
development:
clientId:
$secret:
env: AUTH_GITHUB_CLIENT_ID
$env: AUTH_GITHUB_CLIENT_ID
clientSecret:
$secret:
env: AUTH_GITHUB_CLIENT_SECRET
$env: AUTH_GITHUB_CLIENT_SECRET
enterpriseInstanceUrl:
$secret:
env: AUTH_GITHUB_ENTERPRISE_INSTANCE_URL
$env: AUTH_GITHUB_ENTERPRISE_INSTANCE_URL
gitlab:
development:
clientId:
$secret:
env: AUTH_GITLAB_CLIENT_ID
$env: AUTH_GITLAB_CLIENT_ID
clientSecret:
$secret:
env: AUTH_GITLAB_CLIENT_SECRET
$env: AUTH_GITLAB_CLIENT_SECRET
audience:
$secret:
env: GITLAB_BASE_URL
$env: GITLAB_BASE_URL
saml:
entryPoint: 'http://localhost:7001/'
issuer: 'passport-saml'
okta:
development:
clientId:
$secret:
env: AUTH_OKTA_CLIENT_ID
$env: AUTH_OKTA_CLIENT_ID
clientSecret:
$secret:
env: AUTH_OKTA_CLIENT_SECRET
$env: AUTH_OKTA_CLIENT_SECRET
audience:
$secret:
env: AUTH_OKTA_AUDIENCE
$env: AUTH_OKTA_AUDIENCE
oauth2:
development:
clientId:
$secret:
env: AUTH_OAUTH2_CLIENT_ID
$env: AUTH_OAUTH2_CLIENT_ID
clientSecret:
$secret:
env: AUTH_OAUTH2_CLIENT_SECRET
$env: AUTH_OAUTH2_CLIENT_SECRET
authorizationUrl:
$secret:
env: AUTH_OAUTH2_AUTH_URL
$env: AUTH_OAUTH2_AUTH_URL
tokenUrl:
$secret:
env: AUTH_OAUTH2_TOKEN_URL
$env: AUTH_OAUTH2_TOKEN_URL
auth0:
development:
clientId:
$secret:
env: AUTH_AUTH0_CLIENT_ID
$env: AUTH_AUTH0_CLIENT_ID
clientSecret:
$secret:
env: AUTH_AUTH0_CLIENT_SECRET
$env: AUTH_AUTH0_CLIENT_SECRET
domain:
$secret:
env: AUTH_AUTH0_DOMAIN
$env: AUTH_AUTH0_DOMAIN
microsoft:
development:
clientId:
$secret:
env: AUTH_MICROSOFT_CLIENT_ID
$env: AUTH_MICROSOFT_CLIENT_ID
clientSecret:
$secret:
env: AUTH_MICROSOFT_CLIENT_SECRET
$env: AUTH_MICROSOFT_CLIENT_SECRET
tenantId:
$secret:
env: AUTH_MICROSOFT_TENANT_ID
$env: AUTH_MICROSOFT_TENANT_ID
costInsights:
engineerCost: 200000
products:
docs/auth/auth-backend-classes.md
+6 -11
View File
@@ -67,26 +67,21 @@ auth:
google:
development:
clientId:
$secret:
env: AUTH_GOOGLE_CLIENT_ID
$env: AUTH_GOOGLE_CLIENT_ID
clientSecret:
$secret:
env: AUTH_GOOGLE_CLIENT_SECRET
$env: AUTH_GOOGLE_CLIENT_SECRET
github:
development:
clientId:
$secret:
env: AUTH_GITHUB_CLIENT_ID
$env: AUTH_GITHUB_CLIENT_ID
clientSecret:
$secret:
env: AUTH_GITHUB_CLIENT_SECRET
$env: AUTH_GITHUB_CLIENT_SECRET
enterpriseInstanceUrl:
$secret:
env: AUTH_GITHUB_ENTERPRISE_INSTANCE_URL
$env: AUTH_GITHUB_ENTERPRISE_INSTANCE_URL
gitlab:
development:
clientId:
$secret:
$env:
...
```
docs/conf/index.md
+2 -1
View File
@@ -18,7 +18,8 @@ allowing for customization.
Configuration is stored in `app-config.yaml` files, with support for suffixes
such as `app-config.production.yaml` to override values for specific
environments. The configuration files themselves contain plain YAML, but with
support for loading in secrets from various sources using a `$secret` key.
support for loading in secrets from various sources using for example `$env` and
`$file` keys.
It is also possible to supply configuration through environment variables, for
example `APP_CONFIG_app_baseUrl=https://staging.example.com`. However these
docs/conf/writing.md
+14 -19
View File
@@ -90,18 +90,17 @@ order:
## Secrets
Secrets are supported via a special `$secret` key, which in turn provides a
number of different ways to read in secrets. To load a configuration value as a
secret, supply an object with a single `$secret` key, and within that supply an
object that describes how the secret is loaded. For example, the following will
read the config key `backend.mySecretKey` from the environment variable
`MY_SECRET_KEY`:
Secrets are supported via a special secret keys that are prefixed with `$`,
which in turn provides a number of different ways to read in secrets. To load a
configuration value as a secret, supply an object with one of the special secret
keys, for example `$env` or `$file`. A full list of supported secret keys can be
found below. For example, the following will read the config key
`backend.mySecretKey` from the environment variable `MY_SECRET_KEY`:
```yaml
backend:
mySecretKey:
$secret:
env: MY_SECRET_KEY
$env: MY_SECRET_KEY
```
With the above configuration, calling `config.getString('backend.mySecretKey')`
@@ -123,8 +122,7 @@ This reads a secret from an environment variable. For example, the following
config loads the secret from the `MY_SECRET` env var.
```yaml
$secret:
env: MY_SECRET
$env: MY_SECRET
```
### File Secrets
@@ -135,22 +133,19 @@ following reads the contents of `my-secret.txt` relative to the config file
itself:
```yaml
$secret:
file: ./my-secret.txt
$file: ./my-secret.txt
```
### Data File Secrets
This reads secrets from a path within a JSON-like data file. The file path
behaves similar to file secrets, but in addition a `path` is used to point to a
specific value inside the file. Supported file extensions are `.json`, `.yaml`,
and `.yml`. For example, the following would read out `my-secret-key` from
`my-secrets.json`:
behaves similar to file secrets, but with the addition of a url fragment that is
used to point to a specific value inside the file. Supported file extensions are
`.json`, `.yaml`, and `.yml`. For example, the following would read out
`my-secret-key` from `my-secrets.json`:
```yaml
$secret:
data: ./my-secrets.json
path: deployment.key
$data: ./my-secrets.json#deployment.key
# my-secrets.json
{
docs/features/software-catalog/configuration.md
+6 -12
View File
@@ -23,32 +23,26 @@ integrations:
github:
- host: github.com
token:
$secret:
env: GITHUB_TOKEN
$env: GITHUB_TOKEN
- host: ghe.example.net
apiBaseUrl: https://ghe.example.net/api/v3
rawBaseUrl: https://ghe.example.net/raw
token:
$secret:
env: GHE_TOKEN
$env: GHE_TOKEN
gitlab:
- host: gitlab.com
token:
$secret:
env: GITLAB_TOKEN
$env: GITLAB_TOKEN
bitbucket:
- host: bitbucket.org
username:
$secret:
env: BITBUCKET_USERNAME
$env: BITBUCKET_USERNAME
appPassword:
$secret:
env: BITBUCKET_APP_PASSWORD
$env: BITBUCKET_APP_PASSWORD
azure:
- host: dev.azure.com
token:
$secret:
env: AZURE_TOKEN
$env: AZURE_TOKEN
```
Each key under `integrations` is a separate configuration for each external
docs/features/software-templates/installation.md
+3 -6
View File
@@ -232,15 +232,13 @@ instance:
scaffolder:
github:
token:
$secret:
env: GITHUB_ACCESS_TOKEN
$env: GITHUB_ACCESS_TOKEN
visibility: public # or 'internal' or 'private'
gitlab:
api:
baseUrl: https://gitlab.com
token:
$secret:
env: SCAFFOLDER_GITLAB_PRIVATE_TOKEN
$env: SCAFFOLDER_GITLAB_PRIVATE_TOKEN
```
#### Azure DevOps
@@ -257,8 +255,7 @@ scaffolder:
baseUrl: https://dev.azure.com/{your-organization}
api:
token:
$secret:
env: AZURE_PRIVATE_TOKEN
$env: AZURE_PRIVATE_TOKEN
```
### Running the Backend
docs/plugins/proxying.md
+1 -2
View File
@@ -41,8 +41,7 @@ proxy:
target: http://larger.example.com:8080/svc.v1
headers:
Authorization:
$secret:
env: EXAMPLE_AUTH_HEADER
$env: EXAMPLE_AUTH_HEADER
```
Each key under the proxy configuration entry is a route to match, below the
docs/tutorials/quickstart-app-auth.md
+3 -6
View File
@@ -72,15 +72,12 @@ auth:
github:
development:
clientId:
$secret:
env: AUTH_GITHUB_CLIENT_ID
$env: AUTH_GITHUB_CLIENT_ID
clientSecret:
$secret:
env: AUTH_GITHUB_CLIENT_SECRET
$env: AUTH_GITHUB_CLIENT_SECRET
## uncomment the following three lines if using enterprise
# enterpriseInstanceUrl:
# $secret:
# env: AUTH_GITHUB_ENTERPRISE_INSTANCE_URL
# $env: AUTH_GITHUB_ENTERPRISE_INSTANCE_URL
```
2. Set environment variables in whatever fashion is easiest for you. I chose to
packages/config-loader/src/lib/reader.test.ts
+42 -1
View File
@@ -84,6 +84,47 @@ describe('readConfigFile', () => {
});
it('should read secrets', async () => {
const readFile = memoryFiles({
'./app-config.yaml': 'app: { $file: "./my-secret" }',
});
const readSecret = jest.fn().mockResolvedValue('secret');
const config = readConfigFile('./app-config.yaml', {
...mockContext,
readFile,
readSecret: readSecret as ReadSecretFunc,
});
await expect(config).resolves.toEqual({
data: {
app: 'secret',
},
context: 'app-config.yaml',
});
expect(readSecret).toHaveBeenCalledWith('.app', {
file: './my-secret',
});
});
it('should not allow keys adjacent to secrets', async () => {
const readFile = memoryFiles({
'./app-config.yaml': 'app: { extraKey: 3, $file: "./my-secret" }',
});
const readSecret = jest.fn().mockResolvedValue('secret');
const config = readConfigFile('./app-config.yaml', {
...mockContext,
readFile,
readSecret: readSecret as ReadSecretFunc,
});
await expect(config).rejects.toThrow(
"Secret key '$file' has adjacent keys at .app",
);
expect(readSecret).not.toHaveBeenCalled();
});
it('should read deprecated secrets', async () => {
const readFile = memoryFiles({
'./app-config.yaml': 'app: { $secret: { file: "./my-secret" } }',
});
@@ -106,7 +147,7 @@ describe('readConfigFile', () => {
});
});
it('should require secrets to be objects', async () => {
it('should require deprecated secrets to be objects', async () => {
const readFile = memoryFiles({
'./app-config.yaml': 'app: { $secret: ["wrong-type"] }',
});
packages/config-loader/src/lib/reader.ts
+25 -1
View File
@@ -31,6 +31,8 @@ export async function readConfigFile(
const configYaml = await ctx.readFile(filePath);
const config = yaml.parse(configYaml);
const context = basename(filePath);
async function transform(
obj: JsonValue,
path: string,
@@ -56,7 +58,11 @@ export async function readConfigFile(
return arr;
}
// TODO(Rugvip): This form of declaring secrets is deprecated, warn and remove in the future
if ('$secret' in obj) {
console.warn(
`Deprecated secret declaration at '${path}' in '${context}', use $env, $file, etc. instead`,
);
if (!isObject(obj.$secret)) {
throw TypeError(`Expected object at secret ${path}.$secret`);
}
@@ -68,6 +74,24 @@ export async function readConfigFile(
}
}
// Check if there's any key that starts with a '$', in that case we treat
// this entire object as a secret.
const [secretKey] = Object.keys(obj).filter(key => key.startsWith('$'));
if (secretKey) {
if (Object.keys(obj).length !== 1) {
throw new Error(
`Secret key '${secretKey}' has adjacent keys at ${path}`,
);
}
try {
return await ctx.readSecret(path, {
[secretKey.slice(1)]: obj[secretKey],
});
} catch (error) {
throw new Error(`Invalid secret at ${path}: ${error.message}`);
}
}
const out: JsonObject = {};
for (const [key, value] of Object.entries(obj)) {
@@ -87,5 +111,5 @@ export async function readConfigFile(
if (!isObject(finalConfig)) {
throw new TypeError('Expected object at config root');
}
return { data: finalConfig, context: basename(filePath) };
return { data: finalConfig, context };
}
packages/config-loader/src/lib/secrets.test.ts
+16 -4
View File
@@ -56,21 +56,33 @@ describe('readSecret', () => {
});
it('should read data secrets', async () => {
// Deprecated object form
await expect(
readSecret({ data: 'my-data.json', path: 'a.b.c' }, ctx),
).resolves.toBe('42');
await expect(
readSecret({ data: 'my-data.yaml', path: 'some.yaml.key' }, ctx),
).resolves.toBe('7');
await expect(
readSecret({ data: 'my-data.yml', path: 'different.key' }, ctx),
).resolves.toBe('hello');
await expect(
readSecret({ data: 'no-data.yml', path: 'different.key' }, ctx),
).rejects.toThrow('File not found!');
// New format with path in fragment
await expect(readSecret({ data: 'my-data.json#a.b.c' }, ctx)).resolves.toBe(
'42',
);
await expect(
readSecret({ data: 'my-data.yaml#some.yaml.key' }, ctx),
).resolves.toBe('7');
await expect(
readSecret({ data: 'my-data.yml#different.key' }, ctx),
).resolves.toBe('hello');
await expect(
readSecret({ data: 'no-data.yml#different.key' }, ctx),
).rejects.toThrow('File not found!');
});
it('should reject invalid secrets', async () => {
@@ -84,7 +96,7 @@ describe('readSecret', () => {
"Secret must contain one of 'file', 'env', 'data'",
);
await expect(readSecret({ data: 'no-data.yml' }, ctx)).rejects.toThrow(
'path is a required field',
"Invalid format for data secret value, must be of the form <filepath>#<datapath>, got 'no-data.yml'",
);
await expect(
readSecret({ data: 'no-parser.js', path: '.' }, ctx),
packages/config-loader/src/lib/secrets.ts
+15 -16
View File
@@ -38,9 +38,8 @@ type EnvSecret = {
type DataSecret = {
// Path to the data secret file, relative to the config file.
data: string;
// The path to the value inside the data file.
// Either a '.' separated list, or an array of path segments.
path: string | string[];
// The path to the value inside the data file, each element separated by '.'.
path?: string;
};
type Secret = FileSecret | EnvSecret | DataSecret;
@@ -55,12 +54,6 @@ const secretLoaderSchemas = {
}),
data: yup.object({
data: yup.string().required(),
path: yup.lazy(value => {
if (typeof value === 'string') {
return yup.string().required();
}
return yup.array().of(yup.string().required()).required();
}),
}),
};
@@ -111,24 +104,30 @@ export async function readSecret(
return ctx.env[secret.env];
}
if ('data' in secret) {
const ext = extname(secret.data);
const url =
'path' in secret ? `${secret.data}#${secret.path}` : secret.data;
const [filePath, dataPath] = url.split(/#(.*)/);
if (!dataPath) {
throw new Error(
`Invalid format for data secret value, must be of the form <filepath>#<datapath>, got '${url}'`,
);
}
const ext = extname(filePath);
const parser = dataSecretParser[ext];
if (!parser) {
throw new Error(`No data secret parser available for extension ${ext}`);
}
const content = await ctx.readFile(secret.data);
const content = await ctx.readFile(filePath);
const { path } = secret;
const parts = typeof path === 'string' ? path.split('.') : path;
const parts = dataPath.split('.');
let value: JsonValue | undefined = await parser(content);
for (const [index, part] of parts.entries()) {
if (!isObject(value)) {
const errPath = parts.slice(0, index).join('.');
throw new Error(
`Value is not an object at ${errPath} in ${secret.data}`,
);
throw new Error(`Value is not an object at ${errPath} in ${filePath}`);
}
value = value[part];
}
packages/config-loader/src/loader.test.ts
+1 -2
View File
@@ -24,8 +24,7 @@ describe('loadConfig', () => {
app:
title: Example App
sessionKey:
$secret:
file: secrets/session-key.txt
$file: secrets/session-key.txt
`,
'/root/app-config.development.yaml': `
app:
packages/create-app/templates/default-app/app-config.yaml.hbs
+8 -16
View File
@@ -22,22 +22,17 @@ backend:
client: pg
connection:
host:
$secret:
env: POSTGRES_HOST
$env: POSTGRES_HOST
port:
$secret:
env: POSTGRES_PORT
$env: POSTGRES_PORT
user:
$secret:
env: POSTGRES_USER
$env: POSTGRES_USER
password:
$secret:
env: POSTGRES_PASSWORD
$env: POSTGRES_PASSWORD
# https://node-postgres.com/features/ssl
#ssl: require # see https://www.postgresql.org/docs/current/libpq-ssl.html Table 33.1. SSL Mode Descriptions (e.g. require)
#ca: # if you have a CA file and want to verify it you can uncomment this section
# $secret:
# file: <file-path>/ca/server.crt
# $file: <file-path>/ca/server.crt
{{/if}}
proxy:
@@ -60,8 +55,7 @@ auth:
scaffolder:
github:
token:
$secret:
env: GITHUB_ACCESS_TOKEN
$env: GITHUB_ACCESS_TOKEN
visibility: public # or 'internal' or 'private'
catalog:
@@ -72,14 +66,12 @@ catalog:
providers:
- target: https://github.com
token:
$secret:
env: GITHUB_PRIVATE_TOKEN
$env: GITHUB_PRIVATE_TOKEN
# Example for how to add your GitHub Enterprise instance:
# - target: https://ghe.example.net
# apiBaseUrl: https://ghe.example.net/api/v3
# token:
# $secret:
# env: GHE_PRIVATE_TOKEN
# $env: GHE_PRIVATE_TOKEN
locations:
# Backstage example components
- type: url
plugins/circleci/README.md
+1 -2
View File
@@ -57,8 +57,7 @@ proxy:
target: https://circleci.com/api/v1.1
headers:
Circle-Token:
$secret:
env: CIRCLECI_AUTH_TOKEN
$env: CIRCLECI_AUTH_TOKEN
```
5. Get and provide `CIRCLECI_AUTH_TOKEN` as env variable (https://circleci.com/docs/api/#add-an-api-token)
plugins/jenkins/README.md
+1 -2
View File
@@ -30,8 +30,7 @@ proxy:
changeOrigin: true
headers:
Authorization:
$secret:
env: JENKINS_BASIC_AUTH_HEADER
$env: JENKINS_BASIC_AUTH_HEADER
```
4. Add an environment variable which contains the Jenkins credentials, (note: use an API token not your password)
plugins/rollbar-backend/README.md
+1 -2
View File
@@ -10,8 +10,7 @@ The following values are read from the configuration file.
rollbar:
organization: spotify
accountToken:
$secret:
env: ROLLBAR_ACCOUNT_TOKEN
$env: ROLLBAR_ACCOUNT_TOKEN
```
_NOTE: The `ROLLBAR_ACCOUNT_TOKEN` environment variable must be set to a read
plugins/rollbar/README.md
+1 -2
View File
@@ -55,8 +55,7 @@ const ServiceEntityPage = ({ entity }: { entity: Entity }) => (
rollbar:
organization: spotify
accountToken:
$secret:
env: ROLLBAR_ACCOUNT_TOKEN
$env: ROLLBAR_ACCOUNT_TOKEN
```
7. Annotate entities with the rollbar project slug