config-loader: collect visibility paths both by data and schema path

Signed-off-by: Patrik Oldsberg <poldsberg@gmail.com>

packages/config-loader/package.json

@@ -38,6 +38,7 @@
     "fs-extra": "9.1.0",
     "json-schema": "^0.3.0",
     "json-schema-merge-allof": "^0.8.1",
+    "json-schema-traverse": "^1.0.0",
     "typescript-json-schema": "^0.50.1",
     "yaml": "^1.9.2",
     "yup": "^0.32.9"

packages/config-loader/src/lib/schema/compile.test.ts

@@ -30,11 +30,13 @@ describe('compileConfigSchemas', () => {
     ]);
     expect(validate([{ data: { a: 1 }, context: 'test' }])).toEqual({
       errors: ['Config should be string { type=string } at /a'],
-      visibilityByPath: new Map(),
+      visibilityByDataPath: new Map(),
+      visibilityBySchemaPath: new Map(),
     });
     expect(validate([{ data: { b: 'b' }, context: 'test' }])).toEqual({
       errors: ['Config should be number { type=number } at /b'],
-      visibilityByPath: new Map(),
+      visibilityByDataPath: new Map(),
+      visibilityBySchemaPath: new Map(),
     });
   });
 
@@ -78,7 +80,7 @@ describe('compileConfigSchemas', () => {
         { data: { a: 'a', b: 'b', c: 'c', d: ['d'] }, context: 'test' },
       ]),
     ).toEqual({
-      visibilityByPath: new Map(
+      visibilityByDataPath: new Map(
         Object.entries({
           '/a': 'frontend',
           '/b': 'secret',
@@ -86,6 +88,14 @@ describe('compileConfigSchemas', () => {
           '/d/0': 'frontend',
         }),
       ),
+      visibilityBySchemaPath: new Map(
+        Object.entries({
+          '/properties/a': 'frontend',
+          '/properties/b': 'secret',
+          '/properties/d': 'secret',
+          '/properties/d/items': 'frontend',
+        }),
+      ),
     });
   });
 

packages/config-loader/src/lib/schema/compile.ts

@@ -17,6 +17,7 @@
 import Ajv from 'ajv';
 import { JSONSchema7 as JSONSchema } from 'json-schema';
 import mergeAllOf, { Resolvers } from 'json-schema-merge-allof';
+import traverse from 'json-schema-traverse';
 import { ConfigReader } from '@backstage/config';
 import {
   ConfigSchemaPackageEntry,
@@ -38,7 +39,7 @@ export function compileConfigSchemas(
   // The ajv instance below is stateful and doesn't really allow for additional
   // output during validation. We work around this by having this extra piece
   // of state that we reset before each validation.
-  const visibilityByPath = new Map<string, ConfigVisibility>();
+  const visibilityByDataPath = new Map<string, ConfigVisibility>();
 
   const ajv = new Ajv({
     allErrors: true,
@@ -62,7 +63,7 @@ export function compileConfigSchemas(
             /\['?(.*?)'?\]/g,
             (_, segment) => `/${segment}`,
           );
-          visibilityByPath.set(normalizedPath, visibility);
+          visibilityByDataPath.set(normalizedPath, visibility);
         }
         return true;
       };
@@ -80,10 +81,17 @@ export function compileConfigSchemas(
   const merged = mergeConfigSchemas(schemas.map(_ => _.value));
   const validate = ajv.compile(merged);
 
+  const visibilityBySchemaPath = new Map<string, ConfigVisibility>();
+  traverse(merged, (schema, path) => {
+    if (schema.visibility && schema.visibility !== 'backend') {
+      visibilityBySchemaPath.set(path, schema.visibility);
+    }
+  });
+
   return configs => {
     const config = ConfigReader.fromConfigs(configs).get();
 
-    visibilityByPath.clear();
+    visibilityByDataPath.clear();
 
     const valid = validate(config);
     if (!valid) {
@@ -95,12 +103,14 @@ export function compileConfigSchemas(
             .join(' ');
           return `Config ${message || ''} { ${paramStr} } at ${dataPath}`;
         }),
-        visibilityByPath: new Map(),
+        visibilityByDataPath: new Map(),
+        visibilityBySchemaPath: new Map(),
       };
     }
 
     return {
-      visibilityByPath: new Map(visibilityByPath),
+      visibilityByDataPath: new Map(visibilityByDataPath),
+      visibilityBySchemaPath,
     };
   };
 }

packages/config-loader/src/lib/schema/filtering.ts

@@ -28,7 +28,7 @@ import {
 export function filterByVisibility(
   data: JsonObject,
   includeVisibilities: ConfigVisibility[],
-  visibilityByPath: Map<string, ConfigVisibility>,
+  visibilityByDataPath: Map<string, ConfigVisibility>,
   transformFunc?: TransformFunc<number | string | boolean>,
   withFilteredKeys?: boolean,
 ): { data: JsonObject; filteredKeys?: string[] } {
@@ -40,7 +40,7 @@ export function filterByVisibility(
     filterPath: string, // Matches the format of the ConfigReader
   ): JsonValue | undefined {
     const visibility =
-      visibilityByPath.get(visibilityPath) ?? DEFAULT_CONFIG_VISIBILITY;
+      visibilityByDataPath.get(visibilityPath) ?? DEFAULT_CONFIG_VISIBILITY;
     const isVisible = includeVisibilities.includes(visibility);
 
     if (typeof jsonVal !== 'object') {

packages/config-loader/src/lib/schema/load.ts

@@ -83,7 +83,7 @@ export async function loadConfigSchema(
           ...filterByVisibility(
             data,
             visibility,
-            result.visibilityByPath,
+            result.visibilityByDataPath,
             valueTransform,
             withFilteredKeys,
           ),
@@ -94,7 +94,7 @@ export async function loadConfigSchema(
           ...filterByVisibility(
             data,
             Array.from(CONFIG_VISIBILITIES),
-            result.visibilityByPath,
+            result.visibilityByDataPath,
             valueTransform,
             withFilteredKeys,
           ),

packages/config-loader/src/lib/schema/types.ts

@@ -65,7 +65,14 @@ type ValidationResult = {
    *
    * The path in the key uses the form `/<key>/<sub-key>/<array-index>/<leaf-key>`
    */
-  visibilityByPath: Map<string, ConfigVisibility>;
+  visibilityByDataPath: Map<string, ConfigVisibility>;
+
+  /**
+   * The configuration visibilities that were discovered during validation.
+   *
+   * The path in the key uses the form `/properties/<key>/items/additionalProperties/<leaf-key>`
+   */
+  visibilityBySchemaPath: Map<string, ConfigVisibility>;
 };
 
 /**