Merge branch 'master' of github.com:backstage/backstage into rugvip/exts

* 'master' of github.com:backstage/backstage:
  docs: show how to assign annotations to primitive array item types in config schema
  Add missing IncludeSecret type
  Update .changeset/friendly-numbers-accept.md
  Add documentation for config $include
  Added changeset for config $include
  Add deprecation notice to $data
  Config: Add $include which replaces $data and can include any value

.changeset/friendly-numbers-accept.md

@@ -0,0 +1,20 @@
+---
+'@backstage/config-loader': patch
+---
+
+Deprecate `$data` and replace it with `$include` which allows for any type of json value to be read from external files. In addition, `$include` can be used without a path, which causes the value at the root of the file to be loaded.
+
+Most usages of `$data` can be directly replaced with `$include`, except if the referenced value is not a string, in which case the value needs to be changed. For example:
+
+```yaml
+# app-config.yaml
+foo:
+  $data: foo.yaml#myValue # replacing with $include will turn the value into a number
+  $data: bar.yaml#myValue # replacing with $include is safe
+
+# foo.yaml
+myValue: 0xf00
+
+# bar.yaml
+myValue: bar
+```

docs/conf/defining.md

@@ -36,6 +36,10 @@ export interface Config {
      * @visibility frontend
      */
     baseUrl: string;
+
+    // Use @items.<name> to assign annotations to primitive array items
+    /** @items.visibility frontend */
+    myItems: string[];
   };
 }
 ```

docs/conf/writing.md

@@ -141,16 +141,17 @@ itself:
 $file: ./my-secret.txt
 ```
 
-### Data File Secrets
+### Including Files
 
-This reads secrets from a path within a JSON-like data file. The file path
-behaves similar to file secrets, but with the addition of a url fragment that is
-used to point to a specific value inside the file. Supported file extensions are
-`.json`, `.yaml`, and `.yml`. For example, the following would read out
-`my-secret-key` from `my-secrets.json`:
+The `$include` keyword can be used to load in JSON data from an external file.
+It's able to load and parse data from `.json`, `.yml`, and `.yaml` files. It's
+also possible to include a url fragment (`#`) to point to a value at the given
+path in the file.
+
+For example, the following would read `my-secret-key` from `my-secrets.json`:
 
 ```yaml
-$data: ./my-secrets.json#deployment.key
+$include: ./my-secrets.json#deployment.key
 ```
 
 Example `my-secrets.json` file:

packages/config-loader/src/lib/secrets.test.ts

@@ -28,6 +28,7 @@ const ctx: ReaderContext = {
       'my-data.json': '{"a":{"b":{"c":42}}}',
       'my-data.yaml': 'some:\n yaml:\n  key: 7',
       'my-data.yml': 'different: { key: hello }',
+      'invalid.yaml': 'foo: [}',
     } as { [key: string]: string })[path];
 
     if (!content) {
@@ -84,6 +85,41 @@ describe('readSecret', () => {
     ).rejects.toThrow('File not found!');
   });
 
+  it('should include extra files', async () => {
+    // New format with path in fragment
+    await expect(
+      readSecret({ include: 'my-data.json#a.b.c' }, ctx),
+    ).resolves.toBe(42);
+    await expect(
+      readSecret({ include: 'my-data.json#a.b' }, ctx),
+    ).resolves.toEqual({ c: 42 });
+    await expect(
+      readSecret({ include: 'my-data.yaml#some.yaml.key' }, ctx),
+    ).resolves.toBe(7);
+    await expect(readSecret({ include: 'my-data.yaml' }, ctx)).resolves.toEqual(
+      {
+        some: { yaml: { key: 7 } },
+      },
+    );
+    await expect(
+      readSecret({ include: 'my-data.yaml#' }, ctx),
+    ).resolves.toEqual({
+      some: { yaml: { key: 7 } },
+    });
+    await expect(
+      readSecret({ include: 'my-data.yml#different.key' }, ctx),
+    ).resolves.toBe('hello');
+    await expect(
+      readSecret({ include: 'no-data.yml#different.key' }, ctx),
+    ).rejects.toThrow('File not found!');
+    await expect(
+      readSecret({ include: 'my-data.yml#missing.key' }, ctx),
+    ).rejects.toThrow('Value is not an object at missing in my-data.yml');
+    await expect(readSecret({ include: 'invalid.yaml' }, ctx)).rejects.toThrow(
+      'Failed to parse included file invalid.yaml, YAMLSyntaxError: Flow sequence contains an unexpected }',
+    );
+  });
+
   it('should reject invalid secrets', async () => {
     await expect(readSecret('hello' as any, ctx)).rejects.toThrow(
       'secret must be a `object` type, but the final value was: `"hello"`.',

packages/config-loader/src/lib/secrets.ts

@@ -42,7 +42,12 @@ type DataSecret = {
   path?: string;
 };
 
-type Secret = FileSecret | EnvSecret | DataSecret;
+// TODO(Rugvip): Move this out of secret reading when we remove the deprecated DataSecret and $secret format
+type IncludeSecret = {
+  include: string;
+};
+
+type Secret = FileSecret | EnvSecret | DataSecret | IncludeSecret;
 
 // Schema for each type of secret description
 const secretLoaderSchemas = {
@@ -55,6 +60,9 @@ const secretLoaderSchemas = {
   data: yup.object({
     data: yup.string().required(),
   }),
+  include: yup.object({
+    include: yup.string().required(),
+  }),
 };
 
 // The top-level secret schema, which figures out what type of secret it is.
@@ -94,7 +102,7 @@ const dataSecretParser: {
 export async function readSecret(
   data: JsonObject,
   ctx: ReaderContext,
-): Promise<string | undefined> {
+): Promise<JsonValue | undefined> {
   const secret = secretSchema.validateSync(data, { strict: true }) as Secret;
 
   if ('file' in secret) {
@@ -104,6 +112,9 @@ export async function readSecret(
     return ctx.env[secret.env];
   }
   if ('data' in secret) {
+    console.warn(
+      `Configuration uses deprecated $data key, use $include instead.`,
+    );
     const url =
       'path' in secret ? `${secret.data}#${secret.path}` : secret.data;
     const [filePath, dataPath] = url.split(/#(.*)/);
@@ -134,6 +145,35 @@ export async function readSecret(
 
     return String(value);
   }
+  if ('include' in secret) {
+    const [filePath, dataPath] = secret.include.split(/#(.*)/);
+
+    const ext = extname(filePath);
+    const parser = dataSecretParser[ext];
+    if (!parser) {
+      throw new Error(`No data secret parser available for extension ${ext}`);
+    }
+
+    const content = await ctx.readFile(filePath);
+
+    const parts = dataPath ? dataPath.split('.') : [];
+
+    let value: JsonValue | undefined;
+    try {
+      value = await parser(content);
+    } catch (error) {
+      throw new Error(`Failed to parse included file ${filePath}, ${error}`);
+    }
+    for (const [index, part] of parts.entries()) {
+      if (!isObject(value)) {
+        const errPath = parts.slice(0, index).join('.');
+        throw new Error(`Value is not an object at ${errPath} in ${filePath}`);
+      }
+      value = value[part];
+    }
+
+    return value;
+  }
 
   isNever<typeof secret>();
   throw new Error('Secret was left unhandled');

packages/config-loader/src/lib/types.ts

@@ -14,13 +14,13 @@
  * limitations under the License.
  */
 
-import { JsonObject } from '@backstage/config';
+import { JsonObject, JsonValue } from '@backstage/config';
 
 export type ReadFileFunc = (path: string) => Promise<string>;
 export type ReadSecretFunc = (
   path: string,
   desc: JsonObject,
-) => Promise<string | undefined>;
+) => Promise<JsonValue | undefined>;
 export type SkipFunc = (path: string) => boolean;
 
 /**

packages/config-loader/src/loader.ts

@@ -16,7 +16,7 @@
 
 import fs from 'fs-extra';
 import { resolve as resolvePath, dirname, isAbsolute } from 'path';
-import { AppConfig, JsonObject } from '@backstage/config';
+import { AppConfig, JsonObject, JsonValue } from '@backstage/config';
 import { readConfigFile, readEnvConfig, readSecret } from './lib';
 
 export type LoadConfigOptions = {
@@ -49,7 +49,7 @@ class Context {
   async readSecret(
     _path: string,
     desc: JsonObject,
-  ): Promise<string | undefined> {
+  ): Promise<JsonValue | undefined> {
     return readSecret(desc, this);
   }
 }