From a0214130758e4dfcb27646c605ffbd0dfa5eced1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Fri, 25 Sep 2020 17:06:32 +0200 Subject: [PATCH] feat(catalog-backend): placeholder substitution in entity yaml This feature works the same as $secret does in config - it allows programmatic substitution of values into a document. This is particularly useful e.g. for API type entities where you do not want to repeat your entire API spec document inside the catalog-info.yaml file. For those cases, you can instead do something like ```yaml apiVersion: backstage.io/v1alpha1 kind: API metadata: name: my-federated-service spec: type: graphql definition: $file: ./schema.graphql ``` The textual content of that file will be injected as the value of `definition`, during each refresh loop. Both relative and absolute paths are supported, as well as any HTTP/HTTPS URL pointing to a service that returns the relevant data. --- .../src/ingestion/LocationReaders.ts | 16 +- .../processors/PlaceholderProcessor.test.ts | 364 ++++++++++++++++++ .../processors/PlaceholderProcessor.ts | 214 ++++++++++ .../src/ingestion/processors/types.ts | 2 +- 4 files changed, 588 insertions(+), 8 deletions(-) create mode 100644 plugins/catalog-backend/src/ingestion/processors/PlaceholderProcessor.test.ts create mode 100644 plugins/catalog-backend/src/ingestion/processors/PlaceholderProcessor.ts diff --git a/plugins/catalog-backend/src/ingestion/LocationReaders.ts b/plugins/catalog-backend/src/ingestion/LocationReaders.ts index 11558feae0..14b7f727e4 100644 --- a/plugins/catalog-backend/src/ingestion/LocationReaders.ts +++ b/plugins/catalog-backend/src/ingestion/LocationReaders.ts @@ -15,26 +15,28 @@ */ import { getVoidLogger } from '@backstage/backend-common'; -import { Config, ConfigReader } from '@backstage/config'; import { Entity, EntityPolicies, EntityPolicy, LocationSpec, } from '@backstage/catalog-model'; +import { Config, ConfigReader } from '@backstage/config'; import { Logger } from 'winston'; +import { CatalogRulesEnforcer } from './CatalogRules'; import { AnnotateLocationEntityProcessor } from './processors/AnnotateLocationEntityProcessor'; +import { ApiDefinitionAtLocationProcessor } from './processors/ApiDefinitionAtLocationProcessor'; +import { AzureApiReaderProcessor } from './processors/AzureApiReaderProcessor'; +import { BitbucketApiReaderProcessor } from './processors/BitbucketApiReaderProcessor'; import { EntityPolicyProcessor } from './processors/EntityPolicyProcessor'; import { FileReaderProcessor } from './processors/FileReaderProcessor'; import { GithubReaderProcessor } from './processors/GithubReaderProcessor'; import { GitlabApiReaderProcessor } from './processors/GitlabApiReaderProcessor'; import { GitlabReaderProcessor } from './processors/GitlabReaderProcessor'; -import { BitbucketApiReaderProcessor } from './processors/BitbucketApiReaderProcessor'; -import { AzureApiReaderProcessor } from './processors/AzureApiReaderProcessor'; -import { UrlReaderProcessor } from './processors/UrlReaderProcessor'; import { LocationRefProcessor } from './processors/LocationEntityProcessor'; -import { StaticLocationProcessor } from './processors/StaticLocationProcessor'; +import { PlaceholderProcessor } from './processors/PlaceholderProcessor'; import * as result from './processors/results'; +import { StaticLocationProcessor } from './processors/StaticLocationProcessor'; import { LocationProcessor, LocationProcessorDataResult, @@ -44,10 +46,9 @@ import { LocationProcessorLocationResult, LocationProcessorResult, } from './processors/types'; +import { UrlReaderProcessor } from './processors/UrlReaderProcessor'; import { YamlProcessor } from './processors/YamlProcessor'; import { LocationReader, ReadLocationResult } from './types'; -import { CatalogRulesEnforcer } from './CatalogRules'; -import { ApiDefinitionAtLocationProcessor } from './processors/ApiDefinitionAtLocationProcessor'; // The max amount of nesting depth of generated work items const MAX_DEPTH = 10; @@ -86,6 +87,7 @@ export class LocationReaders implements LocationReader { new AzureApiReaderProcessor(config), new UrlReaderProcessor(), new YamlProcessor(), + PlaceholderProcessor.default(), new ApiDefinitionAtLocationProcessor(), new EntityPolicyProcessor(entityPolicy), new LocationRefProcessor(), diff --git a/plugins/catalog-backend/src/ingestion/processors/PlaceholderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/PlaceholderProcessor.test.ts new file mode 100644 index 0000000000..7e6fa528f1 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/PlaceholderProcessor.test.ts @@ -0,0 +1,364 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Entity } from '@backstage/catalog-model'; +import { + jsonPlaceholderResolver, + PlaceholderProcessor, + PlaceholderResolver, + ResolverParams, + yamlPlaceholderResolver, +} from './PlaceholderProcessor'; +import { LocationProcessorRead } from './types'; + +describe('PlaceholderProcessor', () => { + it('returns placeholder-free data unchanged', async () => { + const input: Entity = { + apiVersion: 'a', + kind: 'k', + metadata: { name: 'n' }, + }; + const processor = new PlaceholderProcessor({ + foo: async () => 'replaced', + }); + await expect( + processor.processEntity( + input, + { type: 't', target: 'l' }, + jest.fn(), + jest.fn(), + ), + ).resolves.toBe(input); + }); + + it('replaces placeholders deep in the data', async () => { + const emit = jest.fn(); + const read = jest.fn(); + const upperResolver: PlaceholderResolver = jest.fn(async ({ value }) => + value!.toString().toUpperCase(), + ); + const processor = new PlaceholderProcessor({ + upper: upperResolver, + }); + + await expect( + processor.processEntity( + { + apiVersion: 'a', + kind: 'k', + metadata: { name: 'n' }, + spec: { a: [{ b: { $upper: 'text' } }] }, + }, + { type: 'fake', target: 'http://example.com' }, + emit, + read, + ), + ).resolves.toEqual({ + apiVersion: 'a', + kind: 'k', + metadata: { name: 'n' }, + spec: { a: [{ b: 'TEXT' }] }, + }); + + expect(emit).not.toBeCalled(); + expect(read).not.toBeCalled(); + expect(upperResolver).toBeCalledWith({ + key: 'upper', + value: 'text', + location: { type: 'fake', target: 'http://example.com' }, + read, + }); + }); + + it('rejects multiple placeholders', async () => { + const emit = jest.fn(); + const read: LocationProcessorRead = jest.fn(); + const processor = new PlaceholderProcessor({ + foo: jest.fn(), + bar: jest.fn(), + }); + + await expect( + processor.processEntity( + { + apiVersion: 'a', + kind: 'k', + metadata: { name: 'n', x: { $foo: 'a', $bar: 'b' } }, + }, + { type: 'a', target: 'b' }, + emit, + read, + ), + ).rejects.toThrow( + 'Placeholders have to be on the form of a single $-prefixed key in an object', + ); + + expect(emit).not.toBeCalled(); + expect(read).not.toBeCalled(); + }); + + it('rejects unknown placeholders', async () => { + const emit = jest.fn(); + const read: LocationProcessorRead = jest.fn(); + const processor = new PlaceholderProcessor({ + bar: jest.fn(), + }); + + await expect( + processor.processEntity( + { + apiVersion: 'a', + kind: 'k', + metadata: { name: 'n', x: { $foo: 'a' } }, + }, + { type: 'a', target: 'b' }, + emit, + read, + ), + ).rejects.toThrow('Encountered unknown placeholder $foo'); + + expect(emit).not.toBeCalled(); + expect(read).not.toBeCalled(); + }); + + it('has builtin text support', async () => { + const emit = jest.fn(); + const read: LocationProcessorRead = jest + .fn() + .mockImplementation(async location => ({ + type: 'data', + location, + data: Buffer.from('TEXT', 'utf-8'), + })); + const processor = PlaceholderProcessor.default(); + + await expect( + processor.processEntity( + { + apiVersion: 'a', + kind: 'k', + metadata: { name: 'n' }, + spec: { data: { $text: '../file.txt' } }, + }, + { + type: 'github', + target: 'https://github.com/spotify/backstage/a/b/catalog-info.yaml', + }, + emit, + read, + ), + ).resolves.toEqual({ + apiVersion: 'a', + kind: 'k', + metadata: { name: 'n' }, + spec: { data: 'TEXT' }, + }); + + expect(emit).not.toBeCalled(); + expect(read).toBeCalledWith({ + type: 'github', + target: 'https://github.com/spotify/backstage/a/file.txt', + }); + }); + + it('has builtin json support', async () => { + const emit = jest.fn(); + const read: LocationProcessorRead = jest + .fn() + .mockImplementation(async location => ({ + type: 'data', + location, + data: Buffer.from(JSON.stringify({ a: ['b', 7] }), 'utf-8'), + })); + const processor = PlaceholderProcessor.default(); + + await expect( + processor.processEntity( + { + apiVersion: 'a', + kind: 'k', + metadata: { name: 'n' }, + spec: { data: { $json: './file.json' } }, + }, + { + type: 'github', + target: 'https://github.com/spotify/backstage/a/b/catalog-info.yaml', + }, + emit, + read, + ), + ).resolves.toEqual({ + apiVersion: 'a', + kind: 'k', + metadata: { name: 'n' }, + spec: { data: { a: ['b', 7] } }, + }); + + expect(emit).not.toBeCalled(); + expect(read).toBeCalledWith({ + type: 'github', + target: 'https://github.com/spotify/backstage/a/b/file.json', + }); + }); + + it('has builtin yaml support', async () => { + const emit = jest.fn(); + const read: LocationProcessorRead = jest + .fn() + .mockImplementation(async location => ({ + type: 'data', + location, + data: Buffer.from('foo:\n - bar: 7', 'utf-8'), + })); + const processor = PlaceholderProcessor.default(); + + await expect( + processor.processEntity( + { + apiVersion: 'a', + kind: 'k', + metadata: { name: 'n' }, + spec: { data: { $yaml: '../file.yaml' } }, + }, + { + type: 'github', + target: 'https://github.com/spotify/backstage/a/b/catalog-info.yaml', + }, + emit, + read, + ), + ).resolves.toEqual({ + apiVersion: 'a', + kind: 'k', + metadata: { name: 'n' }, + spec: { data: { foo: [{ bar: 7 }] } }, + }); + + expect(emit).not.toBeCalled(); + expect(read).toBeCalledWith({ + type: 'github', + target: 'https://github.com/spotify/backstage/a/file.yaml', + }); + }); +}); + +describe('yamlPlaceholderResolver', () => { + let read: jest.MockedFunction; + let params: ResolverParams; + + beforeEach(() => { + read = jest.fn(); + params = { + key: 'a', + value: './file.yaml', + location: { + type: 'github', + target: 'https://github.com/spotify/backstage/a/b/catalog-info.yaml', + }, + read, + }; + }); + + it('parses valid yaml', async () => { + read.mockImplementation(async location => ({ + type: 'data', + location, + data: Buffer.from('foo:\n - bar: 7', 'utf-8'), + })); + + await expect(yamlPlaceholderResolver(params)).resolves.toEqual({ + foo: [{ bar: 7 }], + }); + }); + + it('rejects invalid yaml', async () => { + read.mockImplementation(async location => ({ + type: 'data', + location, + data: Buffer.from('a: 1\n----\n', 'utf-8'), + })); + + await expect(yamlPlaceholderResolver(params)).rejects.toThrow( + 'Placeholder $a found an error in the data at ./file.yaml, YAMLSemanticError: Implicit map keys need to be followed by map values', + ); + }); + + it('rejects multi-document yaml', async () => { + read.mockImplementation(async location => ({ + type: 'data', + location, + data: Buffer.from('foo: 1\n---\nbar: 2\n', 'utf-8'), + })); + + await expect(yamlPlaceholderResolver(params)).rejects.toThrow( + 'Placeholder $a expected to find exactly one document of data at ./file.yaml, found 2', + ); + }); + + it('parses valid json', async () => { + read.mockImplementation(async location => ({ + type: 'data', + location, + data: Buffer.from(JSON.stringify({ a: ['b', 7] }), 'utf-8'), + })); + + await expect(yamlPlaceholderResolver(params)).resolves.toEqual({ + a: ['b', 7], + }); + }); +}); + +describe('jsonPlaceholderResolver', () => { + let read: jest.MockedFunction; + let params: ResolverParams; + + beforeEach(() => { + read = jest.fn(); + params = { + key: 'a', + value: './file.json', + location: { + type: 'github', + target: 'https://github.com/spotify/backstage/a/b/catalog-info.yaml', + }, + read, + }; + }); + + it('parses valid json', async () => { + read.mockImplementation(async location => ({ + type: 'data', + location, + data: Buffer.from(JSON.stringify({ a: ['b', 7] }), 'utf-8'), + })); + + await expect(jsonPlaceholderResolver(params)).resolves.toEqual({ + a: ['b', 7], + }); + }); + + it('rejects invalid json', async () => { + read.mockImplementation(async location => ({ + type: 'data', + location, + data: Buffer.from('}', 'utf-8'), + })); + + await expect(jsonPlaceholderResolver(params)).rejects.toThrow( + 'Placeholder $a failed to parse JSON data at ./file.json, SyntaxError: Unexpected token } in JSON at position 0', + ); + }); +}); diff --git a/plugins/catalog-backend/src/ingestion/processors/PlaceholderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/PlaceholderProcessor.ts new file mode 100644 index 0000000000..e405e5a5df --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/PlaceholderProcessor.ts @@ -0,0 +1,214 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Entity, LocationSpec } from '@backstage/catalog-model'; +import { JsonValue } from '@backstage/config'; +import yaml from 'yaml'; +import { + LocationProcessor, + LocationProcessorEmit, + LocationProcessorRead, +} from './types'; + +export type ResolverParams = { + key: string; + value: JsonValue; + location: LocationSpec; + read: LocationProcessorRead; +}; + +export type PlaceholderResolver = ( + params: ResolverParams, +) => Promise; + +/** + * Traverses raw entity JSON looking for occurrences of $-prefixed placeholders + * that it then fills in with actual data. + */ +export class PlaceholderProcessor implements LocationProcessor { + static default() { + return new PlaceholderProcessor({ + json: jsonPlaceholderResolver, + yaml: yamlPlaceholderResolver, + text: textPlaceholderResolver, + }); + } + + constructor( + private readonly resolvers: Record, + ) {} + + async processEntity( + entity: Entity, + location: LocationSpec, + _emit: LocationProcessorEmit, + read: LocationProcessorRead, + ): Promise { + const process = async (data: any): Promise<[any, boolean]> => { + if (!data || !(data instanceof Object)) { + // Scalars can't have placeholders + return [data, false]; + } + + if (Array.isArray(data)) { + // We're an array - process all entries recursively + const items = await Promise.all(data.map(item => process(item))); + return items.every(([, changed]) => !changed) + ? [data, false] + : [items.map(([item]) => item), true]; + } + + const keys = Object.keys(data); + if (!keys.some(k => k.startsWith('$'))) { + // We're an object but no placeholders at this level - process all + // entries recursively + const entries = await Promise.all( + Object.entries(data).map(([k, v]) => + process(v).then(vp => [k, vp] as const), + ), + ); + return entries.every(([, [, changed]]) => !changed) + ? [data, false] + : [Object.fromEntries(entries.map(([k, [v]]) => [k, v])), true]; + } else if (keys.length !== 1) { + throw new Error( + 'Placeholders have to be on the form of a single $-prefixed key in an object', + ); + } + + const resolverKey = keys[0].substr(1); + const resolver = this.resolvers[resolverKey]; + if (!resolver) { + throw new Error(`Encountered unknown placeholder \$${resolverKey}`); + } + + return [ + await resolver({ + key: resolverKey, + value: data[keys[0]], + location, + read, + }), + true, + ]; + }; + + const [result] = await process(entity); + return result; + } +} + +/* + * Resolvers + */ + +export async function yamlPlaceholderResolver( + params: ResolverParams, +): Promise { + const text = await readTextLocation(params); + + let documents: yaml.Document.Parsed[]; + try { + documents = yaml.parseAllDocuments(text).filter(d => d); + } catch (e) { + throw new Error( + `Placeholder \$${params.key} failed to parse YAML data at ${params.value}, ${e}`, + ); + } + + if (documents.length !== 1) { + throw new Error( + `Placeholder \$${params.key} expected to find exactly one document of data at ${params.value}, found ${documents.length}`, + ); + } + + const document = documents[0]; + + if (document.errors?.length) { + throw new Error( + `Placeholder \$${params.key} found an error in the data at ${params.value}, ${document.errors[0]}`, + ); + } + + return document.toJSON(); +} + +export async function jsonPlaceholderResolver( + params: ResolverParams, +): Promise { + const text = await readTextLocation(params); + + try { + return JSON.parse(text); + } catch (e) { + throw new Error( + `Placeholder \$${params.key} failed to parse JSON data at ${params.value}, ${e}`, + ); + } +} + +export async function textPlaceholderResolver( + params: ResolverParams, +): Promise { + return await readTextLocation(params); +} + +/* + * Helpers + */ + +async function readTextLocation(params: ResolverParams): Promise { + const newLocation = relativeLocation(params); + + try { + const response = await params.read(newLocation); + if (response.type !== 'data') { + throw new Error(`Expected data, got ${response.type}`); + } + return response.data.toString('utf-8'); + } catch (e) { + throw new Error( + `Placeholder \$${params.key} could not read location ${params.value}, ${e}`, + ); + } +} + +function relativeLocation({ + key, + value, + location, +}: ResolverParams): LocationSpec { + if (typeof value !== 'string') { + throw new Error( + `Placeholder \$${key} expected a string value parameter, in the form of an absolute URL or a relative path`, + ); + } + + let url: URL; + try { + // The two-value form of the URL constructor handles relative paths for us + url = new URL(value, location.target); + } catch (e) { + throw new Error( + `Placeholder \$${key} could not form an URL out of ${location.target} and ${value}`, + ); + } + + return { + type: location.type, + target: url.toString(), + }; +} diff --git a/plugins/catalog-backend/src/ingestion/processors/types.ts b/plugins/catalog-backend/src/ingestion/processors/types.ts index 3aaa2d21f9..65e2fdbb0a 100644 --- a/plugins/catalog-backend/src/ingestion/processors/types.ts +++ b/plugins/catalog-backend/src/ingestion/processors/types.ts @@ -52,7 +52,7 @@ export type LocationProcessor = { * @param location The location that the entity came from * @param read Reads the contents of a location * @param emit A sink for auxiliary items resulting from the processing - * @returns The same entity or a modifid version of it + * @returns The same entity or a modified version of it */ processEntity?( entity: Entity,