From 9433565fe7734b17c8577c2740f83e15f3ea0a77 Mon Sep 17 00:00:00 2001 From: Oliver Sand Date: Thu, 24 Sep 2020 18:25:16 +0200 Subject: [PATCH] feat: add ApiDefinitionAtLocationProcessor that allows to load a API definition from another location Resolves #2572 --- .../well-known-annotations.md | 21 +++ packages/catalog-model/examples/all-apis.yaml | 2 + .../catalog-model/examples/spotify-api.yaml | 14 ++ .../src/ingestion/LocationReaders.ts | 30 +++- .../ApiDefinitionAtLocationProcessor.test.ts | 132 ++++++++++++++++++ .../ApiDefinitionAtLocationProcessor.ts | 70 ++++++++++ .../src/ingestion/processors/types.ts | 6 + 7 files changed, 274 insertions(+), 1 deletion(-) create mode 100644 packages/catalog-model/examples/spotify-api.yaml create mode 100644 plugins/catalog-backend/src/ingestion/processors/ApiDefinitionAtLocationProcessor.test.ts create mode 100644 plugins/catalog-backend/src/ingestion/processors/ApiDefinitionAtLocationProcessor.ts diff --git a/docs/features/software-catalog/well-known-annotations.md b/docs/features/software-catalog/well-known-annotations.md index ba5b491e07..e5b66bb3b2 100644 --- a/docs/features/software-catalog/well-known-annotations.md +++ b/docs/features/software-catalog/well-known-annotations.md @@ -41,6 +41,27 @@ expecting a two-item array out of it. The format of the target part is type-dependent and could conceivably even be an empty string, but the separator colon is always present. +### backstage.io/definition-at-location + +```yaml +# Example +apiVersion: backstage.io/v1alpha1 +kind: API +metadata: + name: petstore + annotations: + backstage.io/definition-at-location: 'url:https://petstore.swagger.io/v2/swagger.json' +spec: + type: openapi +``` + +This annotation allows to fetch an API definition from another location, instead +of wrapping the API definition inside the definition field. This allows to +easitly consume existing API definition. The definition is fetched during +ingestion by a processor and included in the entity. It is updated on every +refresh. The annotation contains a location reference string that contains the +location processor type and the target. + ### backstage.io/techdocs-ref ```yaml diff --git a/packages/catalog-model/examples/all-apis.yaml b/packages/catalog-model/examples/all-apis.yaml index 33b000d1ae..3c4f7804bf 100644 --- a/packages/catalog-model/examples/all-apis.yaml +++ b/packages/catalog-model/examples/all-apis.yaml @@ -8,3 +8,5 @@ spec: targets: - https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/hello-world-api.yaml - https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/streetlights-api.yaml + - https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/spotify-api.yaml + - https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/swapi-graphql.yaml diff --git a/packages/catalog-model/examples/spotify-api.yaml b/packages/catalog-model/examples/spotify-api.yaml new file mode 100644 index 0000000000..30524dfdd4 --- /dev/null +++ b/packages/catalog-model/examples/spotify-api.yaml @@ -0,0 +1,14 @@ +apiVersion: backstage.io/v1alpha1 +kind: API +metadata: + name: spotify + description: The Spotify web API + tags: + - spotify + - rest + annotations: + backstage.io/definition-at-location: 'url:https://raw.githubusercontent.com/APIs-guru/openapi-directory/master/APIs/spotify.com/v1/swagger.yaml' +spec: + type: openapi + lifecycle: production + owner: spotify@example.com diff --git a/plugins/catalog-backend/src/ingestion/LocationReaders.ts b/plugins/catalog-backend/src/ingestion/LocationReaders.ts index 8da6e20ab3..11558feae0 100644 --- a/plugins/catalog-backend/src/ingestion/LocationReaders.ts +++ b/plugins/catalog-backend/src/ingestion/LocationReaders.ts @@ -47,6 +47,7 @@ import { import { YamlProcessor } from './processors/YamlProcessor'; import { LocationReader, ReadLocationResult } from './types'; import { CatalogRulesEnforcer } from './CatalogRules'; +import { ApiDefinitionAtLocationProcessor } from './processors/ApiDefinitionAtLocationProcessor'; // The max amount of nesting depth of generated work items const MAX_DEPTH = 10; @@ -85,6 +86,7 @@ export class LocationReaders implements LocationReader { new AzureApiReaderProcessor(config), new UrlReaderProcessor(), new YamlProcessor(), + new ApiDefinitionAtLocationProcessor(), new EntityPolicyProcessor(entityPolicy), new LocationRefProcessor(), new AnnotateLocationEntityProcessor(), @@ -218,7 +220,12 @@ export class LocationReaders implements LocationReader { for (const processor of this.processors) { if (processor.processEntity) { try { - current = await processor.processEntity(current, item.location, emit); + current = await processor.processEntity( + current, + item.location, + emit, + this.readLocation.bind(this), + ); } catch (e) { const message = `Processor ${processor.constructor.name} threw an error while processing entity at ${item.location.type} ${item.location.target}, ${e}`; emit(result.generalError(item.location, message)); @@ -248,4 +255,25 @@ export class LocationReaders implements LocationReader { } } } + + private async readLocation( + location: LocationSpec, + ): Promise { + let locationResult: LocationProcessorResult | undefined; + + await this.handleLocation( + { + type: 'location', + location, + optional: false, + }, + r => (locationResult = r), + ); + + if (!locationResult) { + throw new Error('No location loaded'); + } + + return locationResult; + } } diff --git a/plugins/catalog-backend/src/ingestion/processors/ApiDefinitionAtLocationProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/ApiDefinitionAtLocationProcessor.test.ts new file mode 100644 index 0000000000..ef6d348ad5 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/ApiDefinitionAtLocationProcessor.test.ts @@ -0,0 +1,132 @@ +import { ApiEntity, Entity, LocationSpec } from '@backstage/catalog-model'; +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ApiDefinitionAtLocationProcessor } from './ApiDefinitionAtLocationProcessor'; +import { LocationProcessorResult } from './types'; + +describe('ApiDefinitionAtLocationProcessor', () => { + let processor: ApiDefinitionAtLocationProcessor; + let entity: Entity; + let location: LocationSpec; + + beforeEach(() => { + processor = new ApiDefinitionAtLocationProcessor(); + entity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'API', + metadata: { + name: 'test', + }, + spec: { + lifecycle: 'production', + owner: 'info@example.com', + type: 'openapi', + definition: 'Hello', + }, + }; + location = { + type: 'url', + target: `http://example.com/api.yaml`, + }; + }); + + it('should skip entities without annotation', async () => { + const read = jest.fn( + (): Promise => { + throw new Error(); + }, + ); + + const generated = (await processor.processEntity( + entity, + location, + () => {}, + read, + )) as ApiEntity; + + expect(generated.spec.definition).toBe('Hello'); + }); + + it('should load from location', async () => { + entity.metadata.annotations = { + 'backstage.io/definition-at-location': + 'url:http://example.com/openapi.yaml', + }; + + const read = jest.fn( + (l: LocationSpec): Promise => + Promise.resolve({ + type: 'data', + data: Buffer.from('Hello'), + location: l, + }), + ); + + const generated = (await processor.processEntity( + entity, + location, + () => {}, + read, + )) as ApiEntity; + + expect(generated.spec.definition).toBe('Hello'); + expect(read.mock.calls[0][0]).toStrictEqual({ + type: 'url', + target: 'http://example.com/openapi.yaml', + }); + }); + + it('should throw errors while loading', async () => { + entity.metadata.annotations = { + 'backstage.io/definition-at-location': 'missing', + }; + + const read = jest.fn( + (l: LocationSpec): Promise => + Promise.resolve({ + type: 'error', + error: new Error('Failed to load location'), + location: l, + }), + ); + + await expect( + processor.processEntity(entity, location, () => {}, read), + ).rejects.toThrow('Failed to read location: Failed to load location'); + }); + + it('should throw errors if location read has wrong type', async () => { + entity.metadata.annotations = { + 'backstage.io/definition-at-location': 'wrong', + }; + + const read = jest.fn( + (l: LocationSpec): Promise => + Promise.resolve({ + type: 'location', + optional: false, + location: l, + }), + ); + + await expect( + processor.processEntity(entity, location, () => {}, read), + ).rejects.toThrow( + `Only supports location processor results of type 'data', but got 'location'`, + ); + }); +}); diff --git a/plugins/catalog-backend/src/ingestion/processors/ApiDefinitionAtLocationProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/ApiDefinitionAtLocationProcessor.ts new file mode 100644 index 0000000000..c359e78292 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/ApiDefinitionAtLocationProcessor.ts @@ -0,0 +1,70 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ApiEntity, Entity, LocationSpec } from '@backstage/catalog-model'; +import { + LocationProcessor, + LocationProcessorEmit, + LocationProcessorRead, +} from './types'; + +const DEFINITION_AT_LOCATION_ANNOTATION = 'backstage.io/definition-at-location'; + +export class ApiDefinitionAtLocationProcessor implements LocationProcessor { + async processEntity( + entity: Entity, + _location: LocationSpec, + _emit: LocationProcessorEmit, + read: LocationProcessorRead, + ): Promise { + if ( + entity.kind !== 'API' || + !entity.metadata.annotations || + !entity.metadata.annotations[DEFINITION_AT_LOCATION_ANNOTATION] + ) { + return entity; + } + + const reference = + entity.metadata.annotations[DEFINITION_AT_LOCATION_ANNOTATION]; + const { type, target } = extractReference(reference); + const result = await read({ type, target }); + + if (result.type === 'error') { + throw new Error(`Failed to read location: ${result.error.message}`); + } + + if (result.type !== 'data') { + throw new Error( + `Only supports location processor results of type 'data', but got '${result.type}'`, + ); + } + + const definition = result.data.toString(); + const apiEntity = entity as ApiEntity; + apiEntity.spec.definition = definition; + + return entity; + } +} + +function extractReference(reference: string): { type: string; target: string } { + const delimiterIndex = reference.indexOf(':'); + const type = reference.slice(0, delimiterIndex); + const target = reference.slice(delimiterIndex + 1); + + return { type, target }; +} diff --git a/plugins/catalog-backend/src/ingestion/processors/types.ts b/plugins/catalog-backend/src/ingestion/processors/types.ts index c8f3f6f482..3aaa2d21f9 100644 --- a/plugins/catalog-backend/src/ingestion/processors/types.ts +++ b/plugins/catalog-backend/src/ingestion/processors/types.ts @@ -50,6 +50,7 @@ export type LocationProcessor = { * * @param entity The entity to process * @param location The location that the entity came from + * @param read Reads the contents of a location * @param emit A sink for auxiliary items resulting from the processing * @returns The same entity or a modifid version of it */ @@ -57,6 +58,7 @@ export type LocationProcessor = { entity: Entity, location: LocationSpec, emit: LocationProcessorEmit, + read: LocationProcessorRead, ): Promise; /** @@ -107,3 +109,7 @@ export type LocationProcessorResult = | LocationProcessorDataResult | LocationProcessorEntityResult | LocationProcessorErrorResult; + +export type LocationProcessorRead = ( + location: LocationSpec, +) => Promise;