feat(catalog-backend): placeholder substitution in entity yaml

This feature works the same as $secret does in config - it allows programmatic substitution of values into a document.

This is particularly useful e.g. for API type entities where you do not want to repeat your entire API spec document inside the catalog-info.yaml file. For those cases, you can instead do something like

```yaml
apiVersion: backstage.io/v1alpha1
kind: API
metadata:
  name: my-federated-service
spec:
  type: graphql
  definition:
    $file: ./schema.graphql
```

The textual content of that file will be injected as the value of `definition`, during each refresh loop. Both relative and absolute paths are supported, as well as any HTTP/HTTPS URL pointing to a service that returns the relevant data.

plugins/catalog-backend/src/ingestion/LocationReaders.ts

@@ -15,26 +15,28 @@
  */
 
 import { getVoidLogger } from '@backstage/backend-common';
-import { Config, ConfigReader } from '@backstage/config';
 import {
   Entity,
   EntityPolicies,
   EntityPolicy,
   LocationSpec,
 } from '@backstage/catalog-model';
+import { Config, ConfigReader } from '@backstage/config';
 import { Logger } from 'winston';
+import { CatalogRulesEnforcer } from './CatalogRules';
 import { AnnotateLocationEntityProcessor } from './processors/AnnotateLocationEntityProcessor';
+import { ApiDefinitionAtLocationProcessor } from './processors/ApiDefinitionAtLocationProcessor';
+import { AzureApiReaderProcessor } from './processors/AzureApiReaderProcessor';
+import { BitbucketApiReaderProcessor } from './processors/BitbucketApiReaderProcessor';
 import { EntityPolicyProcessor } from './processors/EntityPolicyProcessor';
 import { FileReaderProcessor } from './processors/FileReaderProcessor';
 import { GithubReaderProcessor } from './processors/GithubReaderProcessor';
 import { GitlabApiReaderProcessor } from './processors/GitlabApiReaderProcessor';
 import { GitlabReaderProcessor } from './processors/GitlabReaderProcessor';
-import { BitbucketApiReaderProcessor } from './processors/BitbucketApiReaderProcessor';
-import { AzureApiReaderProcessor } from './processors/AzureApiReaderProcessor';
-import { UrlReaderProcessor } from './processors/UrlReaderProcessor';
 import { LocationRefProcessor } from './processors/LocationEntityProcessor';
-import { StaticLocationProcessor } from './processors/StaticLocationProcessor';
+import { PlaceholderProcessor } from './processors/PlaceholderProcessor';
 import * as result from './processors/results';
+import { StaticLocationProcessor } from './processors/StaticLocationProcessor';
 import {
   LocationProcessor,
   LocationProcessorDataResult,
@@ -44,10 +46,9 @@ import {
   LocationProcessorLocationResult,
   LocationProcessorResult,
 } from './processors/types';
+import { UrlReaderProcessor } from './processors/UrlReaderProcessor';
 import { YamlProcessor } from './processors/YamlProcessor';
 import { LocationReader, ReadLocationResult } from './types';
-import { CatalogRulesEnforcer } from './CatalogRules';
-import { ApiDefinitionAtLocationProcessor } from './processors/ApiDefinitionAtLocationProcessor';
 
 // The max amount of nesting depth of generated work items
 const MAX_DEPTH = 10;
@@ -86,6 +87,7 @@ export class LocationReaders implements LocationReader {
       new AzureApiReaderProcessor(config),
       new UrlReaderProcessor(),
       new YamlProcessor(),
+      PlaceholderProcessor.default(),
       new ApiDefinitionAtLocationProcessor(),
       new EntityPolicyProcessor(entityPolicy),
       new LocationRefProcessor(),

plugins/catalog-backend/src/ingestion/processors/PlaceholderProcessor.test.ts

@@ -0,0 +1,364 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { Entity } from '@backstage/catalog-model';
+import {
+  jsonPlaceholderResolver,
+  PlaceholderProcessor,
+  PlaceholderResolver,
+  ResolverParams,
+  yamlPlaceholderResolver,
+} from './PlaceholderProcessor';
+import { LocationProcessorRead } from './types';
+
+describe('PlaceholderProcessor', () => {
+  it('returns placeholder-free data unchanged', async () => {
+    const input: Entity = {
+      apiVersion: 'a',
+      kind: 'k',
+      metadata: { name: 'n' },
+    };
+    const processor = new PlaceholderProcessor({
+      foo: async () => 'replaced',
+    });
+    await expect(
+      processor.processEntity(
+        input,
+        { type: 't', target: 'l' },
+        jest.fn(),
+        jest.fn(),
+      ),
+    ).resolves.toBe(input);
+  });
+
+  it('replaces placeholders deep in the data', async () => {
+    const emit = jest.fn();
+    const read = jest.fn();
+    const upperResolver: PlaceholderResolver = jest.fn(async ({ value }) =>
+      value!.toString().toUpperCase(),
+    );
+    const processor = new PlaceholderProcessor({
+      upper: upperResolver,
+    });
+
+    await expect(
+      processor.processEntity(
+        {
+          apiVersion: 'a',
+          kind: 'k',
+          metadata: { name: 'n' },
+          spec: { a: [{ b: { $upper: 'text' } }] },
+        },
+        { type: 'fake', target: 'http://example.com' },
+        emit,
+        read,
+      ),
+    ).resolves.toEqual({
+      apiVersion: 'a',
+      kind: 'k',
+      metadata: { name: 'n' },
+      spec: { a: [{ b: 'TEXT' }] },
+    });
+
+    expect(emit).not.toBeCalled();
+    expect(read).not.toBeCalled();
+    expect(upperResolver).toBeCalledWith({
+      key: 'upper',
+      value: 'text',
+      location: { type: 'fake', target: 'http://example.com' },
+      read,
+    });
+  });
+
+  it('rejects multiple placeholders', async () => {
+    const emit = jest.fn();
+    const read: LocationProcessorRead = jest.fn();
+    const processor = new PlaceholderProcessor({
+      foo: jest.fn(),
+      bar: jest.fn(),
+    });
+
+    await expect(
+      processor.processEntity(
+        {
+          apiVersion: 'a',
+          kind: 'k',
+          metadata: { name: 'n', x: { $foo: 'a', $bar: 'b' } },
+        },
+        { type: 'a', target: 'b' },
+        emit,
+        read,
+      ),
+    ).rejects.toThrow(
+      'Placeholders have to be on the form of a single $-prefixed key in an object',
+    );
+
+    expect(emit).not.toBeCalled();
+    expect(read).not.toBeCalled();
+  });
+
+  it('rejects unknown placeholders', async () => {
+    const emit = jest.fn();
+    const read: LocationProcessorRead = jest.fn();
+    const processor = new PlaceholderProcessor({
+      bar: jest.fn(),
+    });
+
+    await expect(
+      processor.processEntity(
+        {
+          apiVersion: 'a',
+          kind: 'k',
+          metadata: { name: 'n', x: { $foo: 'a' } },
+        },
+        { type: 'a', target: 'b' },
+        emit,
+        read,
+      ),
+    ).rejects.toThrow('Encountered unknown placeholder $foo');
+
+    expect(emit).not.toBeCalled();
+    expect(read).not.toBeCalled();
+  });
+
+  it('has builtin text support', async () => {
+    const emit = jest.fn();
+    const read: LocationProcessorRead = jest
+      .fn()
+      .mockImplementation(async location => ({
+        type: 'data',
+        location,
+        data: Buffer.from('TEXT', 'utf-8'),
+      }));
+    const processor = PlaceholderProcessor.default();
+
+    await expect(
+      processor.processEntity(
+        {
+          apiVersion: 'a',
+          kind: 'k',
+          metadata: { name: 'n' },
+          spec: { data: { $text: '../file.txt' } },
+        },
+        {
+          type: 'github',
+          target: 'https://github.com/spotify/backstage/a/b/catalog-info.yaml',
+        },
+        emit,
+        read,
+      ),
+    ).resolves.toEqual({
+      apiVersion: 'a',
+      kind: 'k',
+      metadata: { name: 'n' },
+      spec: { data: 'TEXT' },
+    });
+
+    expect(emit).not.toBeCalled();
+    expect(read).toBeCalledWith({
+      type: 'github',
+      target: 'https://github.com/spotify/backstage/a/file.txt',
+    });
+  });
+
+  it('has builtin json support', async () => {
+    const emit = jest.fn();
+    const read: LocationProcessorRead = jest
+      .fn()
+      .mockImplementation(async location => ({
+        type: 'data',
+        location,
+        data: Buffer.from(JSON.stringify({ a: ['b', 7] }), 'utf-8'),
+      }));
+    const processor = PlaceholderProcessor.default();
+
+    await expect(
+      processor.processEntity(
+        {
+          apiVersion: 'a',
+          kind: 'k',
+          metadata: { name: 'n' },
+          spec: { data: { $json: './file.json' } },
+        },
+        {
+          type: 'github',
+          target: 'https://github.com/spotify/backstage/a/b/catalog-info.yaml',
+        },
+        emit,
+        read,
+      ),
+    ).resolves.toEqual({
+      apiVersion: 'a',
+      kind: 'k',
+      metadata: { name: 'n' },
+      spec: { data: { a: ['b', 7] } },
+    });
+
+    expect(emit).not.toBeCalled();
+    expect(read).toBeCalledWith({
+      type: 'github',
+      target: 'https://github.com/spotify/backstage/a/b/file.json',
+    });
+  });
+
+  it('has builtin yaml support', async () => {
+    const emit = jest.fn();
+    const read: LocationProcessorRead = jest
+      .fn()
+      .mockImplementation(async location => ({
+        type: 'data',
+        location,
+        data: Buffer.from('foo:\n  - bar: 7', 'utf-8'),
+      }));
+    const processor = PlaceholderProcessor.default();
+
+    await expect(
+      processor.processEntity(
+        {
+          apiVersion: 'a',
+          kind: 'k',
+          metadata: { name: 'n' },
+          spec: { data: { $yaml: '../file.yaml' } },
+        },
+        {
+          type: 'github',
+          target: 'https://github.com/spotify/backstage/a/b/catalog-info.yaml',
+        },
+        emit,
+        read,
+      ),
+    ).resolves.toEqual({
+      apiVersion: 'a',
+      kind: 'k',
+      metadata: { name: 'n' },
+      spec: { data: { foo: [{ bar: 7 }] } },
+    });
+
+    expect(emit).not.toBeCalled();
+    expect(read).toBeCalledWith({
+      type: 'github',
+      target: 'https://github.com/spotify/backstage/a/file.yaml',
+    });
+  });
+});
+
+describe('yamlPlaceholderResolver', () => {
+  let read: jest.MockedFunction<LocationProcessorRead>;
+  let params: ResolverParams;
+
+  beforeEach(() => {
+    read = jest.fn();
+    params = {
+      key: 'a',
+      value: './file.yaml',
+      location: {
+        type: 'github',
+        target: 'https://github.com/spotify/backstage/a/b/catalog-info.yaml',
+      },
+      read,
+    };
+  });
+
+  it('parses valid yaml', async () => {
+    read.mockImplementation(async location => ({
+      type: 'data',
+      location,
+      data: Buffer.from('foo:\n  - bar: 7', 'utf-8'),
+    }));
+
+    await expect(yamlPlaceholderResolver(params)).resolves.toEqual({
+      foo: [{ bar: 7 }],
+    });
+  });
+
+  it('rejects invalid yaml', async () => {
+    read.mockImplementation(async location => ({
+      type: 'data',
+      location,
+      data: Buffer.from('a: 1\n----\n', 'utf-8'),
+    }));
+
+    await expect(yamlPlaceholderResolver(params)).rejects.toThrow(
+      'Placeholder $a found an error in the data at ./file.yaml, YAMLSemanticError: Implicit map keys need to be followed by map values',
+    );
+  });
+
+  it('rejects multi-document yaml', async () => {
+    read.mockImplementation(async location => ({
+      type: 'data',
+      location,
+      data: Buffer.from('foo: 1\n---\nbar: 2\n', 'utf-8'),
+    }));
+
+    await expect(yamlPlaceholderResolver(params)).rejects.toThrow(
+      'Placeholder $a expected to find exactly one document of data at ./file.yaml, found 2',
+    );
+  });
+
+  it('parses valid json', async () => {
+    read.mockImplementation(async location => ({
+      type: 'data',
+      location,
+      data: Buffer.from(JSON.stringify({ a: ['b', 7] }), 'utf-8'),
+    }));
+
+    await expect(yamlPlaceholderResolver(params)).resolves.toEqual({
+      a: ['b', 7],
+    });
+  });
+});
+
+describe('jsonPlaceholderResolver', () => {
+  let read: jest.MockedFunction<LocationProcessorRead>;
+  let params: ResolverParams;
+
+  beforeEach(() => {
+    read = jest.fn();
+    params = {
+      key: 'a',
+      value: './file.json',
+      location: {
+        type: 'github',
+        target: 'https://github.com/spotify/backstage/a/b/catalog-info.yaml',
+      },
+      read,
+    };
+  });
+
+  it('parses valid json', async () => {
+    read.mockImplementation(async location => ({
+      type: 'data',
+      location,
+      data: Buffer.from(JSON.stringify({ a: ['b', 7] }), 'utf-8'),
+    }));
+
+    await expect(jsonPlaceholderResolver(params)).resolves.toEqual({
+      a: ['b', 7],
+    });
+  });
+
+  it('rejects invalid json', async () => {
+    read.mockImplementation(async location => ({
+      type: 'data',
+      location,
+      data: Buffer.from('}', 'utf-8'),
+    }));
+
+    await expect(jsonPlaceholderResolver(params)).rejects.toThrow(
+      'Placeholder $a failed to parse JSON data at ./file.json, SyntaxError: Unexpected token } in JSON at position 0',
+    );
+  });
+});

plugins/catalog-backend/src/ingestion/processors/PlaceholderProcessor.ts

@@ -0,0 +1,214 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { Entity, LocationSpec } from '@backstage/catalog-model';
+import { JsonValue } from '@backstage/config';
+import yaml from 'yaml';
+import {
+  LocationProcessor,
+  LocationProcessorEmit,
+  LocationProcessorRead,
+} from './types';
+
+export type ResolverParams = {
+  key: string;
+  value: JsonValue;
+  location: LocationSpec;
+  read: LocationProcessorRead;
+};
+
+export type PlaceholderResolver = (
+  params: ResolverParams,
+) => Promise<JsonValue>;
+
+/**
+ * Traverses raw entity JSON looking for occurrences of $-prefixed placeholders
+ * that it then fills in with actual data.
+ */
+export class PlaceholderProcessor implements LocationProcessor {
+  static default() {
+    return new PlaceholderProcessor({
+      json: jsonPlaceholderResolver,
+      yaml: yamlPlaceholderResolver,
+      text: textPlaceholderResolver,
+    });
+  }
+
+  constructor(
+    private readonly resolvers: Record<string, PlaceholderResolver>,
+  ) {}
+
+  async processEntity(
+    entity: Entity,
+    location: LocationSpec,
+    _emit: LocationProcessorEmit,
+    read: LocationProcessorRead,
+  ): Promise<Entity> {
+    const process = async (data: any): Promise<[any, boolean]> => {
+      if (!data || !(data instanceof Object)) {
+        // Scalars can't have placeholders
+        return [data, false];
+      }
+
+      if (Array.isArray(data)) {
+        // We're an array - process all entries recursively
+        const items = await Promise.all(data.map(item => process(item)));
+        return items.every(([, changed]) => !changed)
+          ? [data, false]
+          : [items.map(([item]) => item), true];
+      }
+
+      const keys = Object.keys(data);
+      if (!keys.some(k => k.startsWith('$'))) {
+        // We're an object but no placeholders at this level - process all
+        // entries recursively
+        const entries = await Promise.all(
+          Object.entries(data).map(([k, v]) =>
+            process(v).then(vp => [k, vp] as const),
+          ),
+        );
+        return entries.every(([, [, changed]]) => !changed)
+          ? [data, false]
+          : [Object.fromEntries(entries.map(([k, [v]]) => [k, v])), true];
+      } else if (keys.length !== 1) {
+        throw new Error(
+          'Placeholders have to be on the form of a single $-prefixed key in an object',
+        );
+      }
+
+      const resolverKey = keys[0].substr(1);
+      const resolver = this.resolvers[resolverKey];
+      if (!resolver) {
+        throw new Error(`Encountered unknown placeholder \$${resolverKey}`);
+      }
+
+      return [
+        await resolver({
+          key: resolverKey,
+          value: data[keys[0]],
+          location,
+          read,
+        }),
+        true,
+      ];
+    };
+
+    const [result] = await process(entity);
+    return result;
+  }
+}
+
+/*
+ * Resolvers
+ */
+
+export async function yamlPlaceholderResolver(
+  params: ResolverParams,
+): Promise<JsonValue> {
+  const text = await readTextLocation(params);
+
+  let documents: yaml.Document.Parsed[];
+  try {
+    documents = yaml.parseAllDocuments(text).filter(d => d);
+  } catch (e) {
+    throw new Error(
+      `Placeholder \$${params.key} failed to parse YAML data at ${params.value}, ${e}`,
+    );
+  }
+
+  if (documents.length !== 1) {
+    throw new Error(
+      `Placeholder \$${params.key} expected to find exactly one document of data at ${params.value}, found ${documents.length}`,
+    );
+  }
+
+  const document = documents[0];
+
+  if (document.errors?.length) {
+    throw new Error(
+      `Placeholder \$${params.key} found an error in the data at ${params.value}, ${document.errors[0]}`,
+    );
+  }
+
+  return document.toJSON();
+}
+
+export async function jsonPlaceholderResolver(
+  params: ResolverParams,
+): Promise<JsonValue> {
+  const text = await readTextLocation(params);
+
+  try {
+    return JSON.parse(text);
+  } catch (e) {
+    throw new Error(
+      `Placeholder \$${params.key} failed to parse JSON data at ${params.value}, ${e}`,
+    );
+  }
+}
+
+export async function textPlaceholderResolver(
+  params: ResolverParams,
+): Promise<JsonValue> {
+  return await readTextLocation(params);
+}
+
+/*
+ * Helpers
+ */
+
+async function readTextLocation(params: ResolverParams): Promise<string> {
+  const newLocation = relativeLocation(params);
+
+  try {
+    const response = await params.read(newLocation);
+    if (response.type !== 'data') {
+      throw new Error(`Expected data, got ${response.type}`);
+    }
+    return response.data.toString('utf-8');
+  } catch (e) {
+    throw new Error(
+      `Placeholder \$${params.key} could not read location ${params.value}, ${e}`,
+    );
+  }
+}
+
+function relativeLocation({
+  key,
+  value,
+  location,
+}: ResolverParams): LocationSpec {
+  if (typeof value !== 'string') {
+    throw new Error(
+      `Placeholder \$${key} expected a string value parameter, in the form of an absolute URL or a relative path`,
+    );
+  }
+
+  let url: URL;
+  try {
+    // The two-value form of the URL constructor handles relative paths for us
+    url = new URL(value, location.target);
+  } catch (e) {
+    throw new Error(
+      `Placeholder \$${key} could not form an URL out of ${location.target} and ${value}`,
+    );
+  }
+
+  return {
+    type: location.type,
+    target: url.toString(),
+  };
+}

plugins/catalog-backend/src/ingestion/processors/types.ts

@@ -52,7 +52,7 @@ export type LocationProcessor = {
    * @param location The location that the entity came from
    * @param read Reads the contents of a location
    * @param emit A sink for auxiliary items resulting from the processing
-   * @returns The same entity or a modifid version of it
+   * @returns The same entity or a modified version of it
    */
   processEntity?(
     entity: Entity,