diff --git a/.changeset/friendly-numbers-accept.md b/.changeset/friendly-numbers-accept.md new file mode 100644 index 0000000000..7890ec0c4a --- /dev/null +++ b/.changeset/friendly-numbers-accept.md @@ -0,0 +1,20 @@ +--- +'@backstage/config-loader': patch +--- + +Deprecate `$data` and replace it with `$include` which allows for any type of json value to be read from external files. In addition, `$include` can be used without a path, which causes the value at the root of the file to be loaded. + +Most usages of `$data` can be directly replaced with `$include`, except if the referenced value is not a string, in which case the value needs to be changed. For example: + +```yaml +# app-config.yaml +foo: + $data: foo.yaml#myValue # replacing with $include will turn the value into a number + $data: bar.yaml#myValue # replacing with $include is safe + +# foo.yaml +myValue: 0xf00 + +# bar.yaml +myValue: bar +``` diff --git a/docs/conf/writing.md b/docs/conf/writing.md index f28dd5b2d3..7e8402b6f6 100644 --- a/docs/conf/writing.md +++ b/docs/conf/writing.md @@ -141,16 +141,17 @@ itself: $file: ./my-secret.txt ``` -### Data File Secrets +### Including Files -This reads secrets from a path within a JSON-like data file. The file path -behaves similar to file secrets, but with the addition of a url fragment that is -used to point to a specific value inside the file. Supported file extensions are -`.json`, `.yaml`, and `.yml`. For example, the following would read out -`my-secret-key` from `my-secrets.json`: +The `$include` keyword can be used to load in JSON data from an external file. +It's able to load and parse data from `.json`, `.yml`, and `.yaml` files. It's +also possible to include a url fragment (`#`) to point to a value at the given +path in the file. + +For example, the following would read `my-secret-key` from `my-secrets.json`: ```yaml -$data: ./my-secrets.json#deployment.key +$include: ./my-secrets.json#deployment.key ``` Example `my-secrets.json` file: diff --git a/packages/config-loader/src/lib/secrets.test.ts b/packages/config-loader/src/lib/secrets.test.ts index d80ada193b..189834f3aa 100644 --- a/packages/config-loader/src/lib/secrets.test.ts +++ b/packages/config-loader/src/lib/secrets.test.ts @@ -28,6 +28,7 @@ const ctx: ReaderContext = { 'my-data.json': '{"a":{"b":{"c":42}}}', 'my-data.yaml': 'some:\n yaml:\n key: 7', 'my-data.yml': 'different: { key: hello }', + 'invalid.yaml': 'foo: [}', } as { [key: string]: string })[path]; if (!content) { @@ -84,6 +85,41 @@ describe('readSecret', () => { ).rejects.toThrow('File not found!'); }); + it('should include extra files', async () => { + // New format with path in fragment + await expect( + readSecret({ include: 'my-data.json#a.b.c' }, ctx), + ).resolves.toBe(42); + await expect( + readSecret({ include: 'my-data.json#a.b' }, ctx), + ).resolves.toEqual({ c: 42 }); + await expect( + readSecret({ include: 'my-data.yaml#some.yaml.key' }, ctx), + ).resolves.toBe(7); + await expect(readSecret({ include: 'my-data.yaml' }, ctx)).resolves.toEqual( + { + some: { yaml: { key: 7 } }, + }, + ); + await expect( + readSecret({ include: 'my-data.yaml#' }, ctx), + ).resolves.toEqual({ + some: { yaml: { key: 7 } }, + }); + await expect( + readSecret({ include: 'my-data.yml#different.key' }, ctx), + ).resolves.toBe('hello'); + await expect( + readSecret({ include: 'no-data.yml#different.key' }, ctx), + ).rejects.toThrow('File not found!'); + await expect( + readSecret({ include: 'my-data.yml#missing.key' }, ctx), + ).rejects.toThrow('Value is not an object at missing in my-data.yml'); + await expect(readSecret({ include: 'invalid.yaml' }, ctx)).rejects.toThrow( + 'Failed to parse included file invalid.yaml, YAMLSyntaxError: Flow sequence contains an unexpected }', + ); + }); + it('should reject invalid secrets', async () => { await expect(readSecret('hello' as any, ctx)).rejects.toThrow( 'secret must be a `object` type, but the final value was: `"hello"`.', diff --git a/packages/config-loader/src/lib/secrets.ts b/packages/config-loader/src/lib/secrets.ts index 97f0de2940..b0e0a0dd7b 100644 --- a/packages/config-loader/src/lib/secrets.ts +++ b/packages/config-loader/src/lib/secrets.ts @@ -42,7 +42,12 @@ type DataSecret = { path?: string; }; -type Secret = FileSecret | EnvSecret | DataSecret; +// TODO(Rugvip): Move this out of secret reading when we remove the deprecated DataSecret and $secret format +type IncludeSecret = { + include: string; +}; + +type Secret = FileSecret | EnvSecret | DataSecret | IncludeSecret; // Schema for each type of secret description const secretLoaderSchemas = { @@ -55,6 +60,9 @@ const secretLoaderSchemas = { data: yup.object({ data: yup.string().required(), }), + include: yup.object({ + include: yup.string().required(), + }), }; // The top-level secret schema, which figures out what type of secret it is. @@ -94,7 +102,7 @@ const dataSecretParser: { export async function readSecret( data: JsonObject, ctx: ReaderContext, -): Promise { +): Promise { const secret = secretSchema.validateSync(data, { strict: true }) as Secret; if ('file' in secret) { @@ -104,6 +112,9 @@ export async function readSecret( return ctx.env[secret.env]; } if ('data' in secret) { + console.warn( + `Configuration uses deprecated $data key, use $include instead.`, + ); const url = 'path' in secret ? `${secret.data}#${secret.path}` : secret.data; const [filePath, dataPath] = url.split(/#(.*)/); @@ -134,6 +145,35 @@ export async function readSecret( return String(value); } + if ('include' in secret) { + const [filePath, dataPath] = secret.include.split(/#(.*)/); + + const ext = extname(filePath); + const parser = dataSecretParser[ext]; + if (!parser) { + throw new Error(`No data secret parser available for extension ${ext}`); + } + + const content = await ctx.readFile(filePath); + + const parts = dataPath ? dataPath.split('.') : []; + + let value: JsonValue | undefined; + try { + value = await parser(content); + } catch (error) { + throw new Error(`Failed to parse included file ${filePath}, ${error}`); + } + for (const [index, part] of parts.entries()) { + if (!isObject(value)) { + const errPath = parts.slice(0, index).join('.'); + throw new Error(`Value is not an object at ${errPath} in ${filePath}`); + } + value = value[part]; + } + + return value; + } isNever(); throw new Error('Secret was left unhandled'); diff --git a/packages/config-loader/src/lib/types.ts b/packages/config-loader/src/lib/types.ts index e189aef20d..95e1d6655e 100644 --- a/packages/config-loader/src/lib/types.ts +++ b/packages/config-loader/src/lib/types.ts @@ -14,13 +14,13 @@ * limitations under the License. */ -import { JsonObject } from '@backstage/config'; +import { JsonObject, JsonValue } from '@backstage/config'; export type ReadFileFunc = (path: string) => Promise; export type ReadSecretFunc = ( path: string, desc: JsonObject, -) => Promise; +) => Promise; export type SkipFunc = (path: string) => boolean; /** diff --git a/packages/config-loader/src/loader.ts b/packages/config-loader/src/loader.ts index a647367469..11cb1dc338 100644 --- a/packages/config-loader/src/loader.ts +++ b/packages/config-loader/src/loader.ts @@ -16,7 +16,7 @@ import fs from 'fs-extra'; import { resolve as resolvePath, dirname, isAbsolute } from 'path'; -import { AppConfig, JsonObject } from '@backstage/config'; +import { AppConfig, JsonObject, JsonValue } from '@backstage/config'; import { readConfigFile, readEnvConfig, readSecret } from './lib'; export type LoadConfigOptions = { @@ -49,7 +49,7 @@ class Context { async readSecret( _path: string, desc: JsonObject, - ): Promise { + ): Promise { return readSecret(desc, this); } }