From dd995cdc7250709b549a606ddec59827dd12b68b Mon Sep 17 00:00:00 2001 From: Thorsten Hake Date: Thu, 10 Nov 2022 08:54:11 +0100 Subject: [PATCH] Adds support for generic $ref resolving in yaml/json documents. This resolves https://github.com/backstage/backstage/issues/14490. Signed-off-by: Thorsten Hake --- .changeset/tender-colts-greet.md | 5 ++ .../catalog-backend-module-openapi/README.md | 19 ++++-- .../package.json | 2 +- .../src/OpenApiRefProcessor.test.ts | 10 +-- .../src/OpenApiRefProcessor.ts | 4 +- .../src/index.ts | 9 ++- .../src/lib/bundle.test.ts | 68 +++++++++++++++++-- .../src/lib/bundle.ts | 18 ++--- ...test.ts => refPlaceholderResolver.test.ts} | 20 +++--- ...rResolver.ts => refPlaceholderResolver.ts} | 6 +- 10 files changed, 119 insertions(+), 42 deletions(-) create mode 100644 .changeset/tender-colts-greet.md rename plugins/catalog-backend-module-openapi/src/{openApiPlaceholderResolver.test.ts => refPlaceholderResolver.test.ts} (70%) rename plugins/catalog-backend-module-openapi/src/{openApiPlaceholderResolver.ts => refPlaceholderResolver.ts} (94%) diff --git a/.changeset/tender-colts-greet.md b/.changeset/tender-colts-greet.md new file mode 100644 index 0000000000..77c1a8f1b5 --- /dev/null +++ b/.changeset/tender-colts-greet.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend-module-openapi': patch +--- + +Enabled support of resolving `$refs` in all kind of yaml documents, not only OpenAPI. This implicitly adds `$ref` resolving support for AsyncAPI specs. Thus, the `openApiPlaceholderResolver` has been renamed to `refPlaceholderResolver`. diff --git a/plugins/catalog-backend-module-openapi/README.md b/plugins/catalog-backend-module-openapi/README.md index f80798e267..68e9c90e05 100644 --- a/plugins/catalog-backend-module-openapi/README.md +++ b/plugins/catalog-backend-module-openapi/README.md @@ -1,8 +1,10 @@ -# Catalog Backend Module for OpenAPI specifications +# Catalog Backend Module to resolve $refs in yaml documents -This is an extension module to the plugin-catalog-backend plugin, providing extensions targeted at OpenAPI specifications. +This is an extension module to the plugin-catalog-backend plugin, providing an extensions to resolve $refs in yamls documents. -With this you can split your OpenAPI definition into multiple files and reference them. They will be bundled, using an UrlReader, during processing and stored as a single specification. +With this you can split your yaml documents into multiple files and reference them. They will be bundled, using an UrlReader, during processing and stored as a single specification. + +This is useful for OpenAPI and AsyncAPI specifications. ## Installation @@ -15,15 +17,18 @@ yarn add --cwd packages/backend @backstage/plugin-catalog-backend-module-openapi ### Adding the plugin to your `packages/backend` -#### **openApiPlaceholderResolver** +#### **refPlaceholderResolver** -The placeholder resolver can be added by importing `openApiPlaceholderResolver` in `src/plugins/catalog.ts` in your `backend` package and adding the following. +The placeholder resolver can be added by importing `refPlaceholderResolver` in `src/plugins/catalog.ts` in your `backend` package and adding the following. ```ts -builder.setPlaceholderResolver('openapi', openApiPlaceholderResolver); +builder.setPlaceholderResolver('openapi', refPlaceholderResolver); +builder.setPlaceholderResolver('asyncapi', refPlaceholderResolver); ``` -This allows you to use the `$openapi` placeholder when referencing your OpenAPI specification. This will then resolve all `$ref` instances in your specification. +This allows you to use the `$openapi` placeholder when referencing your OpenAPI specification and `$asyncapi` when referencing your AsyncAPI specifications. This will then resolve all `$ref` instances in your specification. + +You can also use this resolver for other kind of yaml files to resolve $ref pointer. ```yaml apiVersion: backstage.io/v1alpha1 diff --git a/plugins/catalog-backend-module-openapi/package.json b/plugins/catalog-backend-module-openapi/package.json index 1055b5ea92..616e10b98d 100644 --- a/plugins/catalog-backend-module-openapi/package.json +++ b/plugins/catalog-backend-module-openapi/package.json @@ -32,7 +32,7 @@ "start": "backstage-cli package start" }, "dependencies": { - "@apidevtools/swagger-parser": "^10.1.0", + "@apidevtools/json-schema-ref-parser": "^9.0.6", "@backstage/backend-common": "workspace:^", "@backstage/catalog-model": "workspace:^", "@backstage/config": "workspace:^", diff --git a/plugins/catalog-backend-module-openapi/src/OpenApiRefProcessor.test.ts b/plugins/catalog-backend-module-openapi/src/OpenApiRefProcessor.test.ts index 9f97e621c2..78568fd19f 100644 --- a/plugins/catalog-backend-module-openapi/src/OpenApiRefProcessor.test.ts +++ b/plugins/catalog-backend-module-openapi/src/OpenApiRefProcessor.test.ts @@ -17,13 +17,13 @@ import { getVoidLogger } from '@backstage/backend-common'; import { ConfigReader } from '@backstage/config'; import { LocationSpec } from '@backstage/plugin-catalog-backend'; import { OpenApiRefProcessor } from './OpenApiRefProcessor'; -import { bundleOpenApiSpecification } from './lib'; +import { bundleFileWithRefs } from './lib'; jest.mock('./lib', () => ({ - bundleOpenApiSpecification: jest.fn(), + bundleFileWithRefs: jest.fn(), })); -const bundledSpecification = ''; +const bundled = ''; describe('OpenApiRefProcessor', () => { const mockLocation = (): LocationSpec => ({ @@ -32,7 +32,7 @@ describe('OpenApiRefProcessor', () => { }); beforeEach(() => { - (bundleOpenApiSpecification as any).mockResolvedValue(bundledSpecification); + (bundleFileWithRefs as any).mockResolvedValue(bundled); }); afterEach(() => { @@ -71,7 +71,7 @@ describe('OpenApiRefProcessor', () => { mockLocation(), ); - expect(result.spec?.definition).toEqual(bundledSpecification); + expect(result.spec?.definition).toEqual(bundled); }); it('should ignore other kinds', async () => { diff --git a/plugins/catalog-backend-module-openapi/src/OpenApiRefProcessor.ts b/plugins/catalog-backend-module-openapi/src/OpenApiRefProcessor.ts index f4cffdf48f..6298aea77d 100644 --- a/plugins/catalog-backend-module-openapi/src/OpenApiRefProcessor.ts +++ b/plugins/catalog-backend-module-openapi/src/OpenApiRefProcessor.ts @@ -21,7 +21,7 @@ import { CatalogProcessor, LocationSpec, } from '@backstage/plugin-catalog-backend'; -import { bundleOpenApiSpecification } from './lib'; +import { bundleFileWithRefs } from './lib'; import { Logger } from 'winston'; /** @@ -84,7 +84,7 @@ export class OpenApiRefProcessor implements CatalogProcessor { this.logger.debug(`Bundling OpenAPI specification from ${location.target}`); try { - const bundledSpec = await bundleOpenApiSpecification( + const bundledSpec = await bundleFileWithRefs( definition.toString(), location.target, this.reader.read, diff --git a/plugins/catalog-backend-module-openapi/src/index.ts b/plugins/catalog-backend-module-openapi/src/index.ts index ccd3ced71e..57f3d8806f 100644 --- a/plugins/catalog-backend-module-openapi/src/index.ts +++ b/plugins/catalog-backend-module-openapi/src/index.ts @@ -13,5 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +import { refPlaceholderResolver } from './refPlaceholderResolver'; + export { OpenApiRefProcessor } from './OpenApiRefProcessor'; -export { openApiPlaceholderResolver } from './openApiPlaceholderResolver'; +export { refPlaceholderResolver } from './refPlaceholderResolver'; +/** + * @public + * @deprecated replaced by refPlaceholderResolver + */ +export const openApiPlaceholderResolver = refPlaceholderResolver; diff --git a/plugins/catalog-backend-module-openapi/src/lib/bundle.test.ts b/plugins/catalog-backend-module-openapi/src/lib/bundle.test.ts index 57ecd54f26..ec77610733 100644 --- a/plugins/catalog-backend-module-openapi/src/lib/bundle.test.ts +++ b/plugins/catalog-backend-module-openapi/src/lib/bundle.test.ts @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { bundleOpenApiSpecification } from './bundle'; +import { bundleFileWithRefs } from './bundle'; const specification = ` openapi: "3.0.0" @@ -70,7 +70,7 @@ paths: type: string `; -describe('bundleOpenApiSpecification', () => { +describe('bundleFileWithRefs', () => { const read = jest.fn(); const resolveUrl = jest.fn(); @@ -81,7 +81,7 @@ describe('bundleOpenApiSpecification', () => { it('should return the bundled specification', async () => { read.mockResolvedValue(list); - const result = await bundleOpenApiSpecification( + const result = await bundleFileWithRefs( specification, 'https://github.com/owner/repo/blob/main/catalog-info.yaml', read, @@ -108,7 +108,7 @@ describe('bundleOpenApiSpecification', () => { read.mockResolvedValue(list); - const result = await bundleOpenApiSpecification( + const result = await bundleFileWithRefs( spec, 'https://github.com/owner/repo/blob/main/catalog-info.yaml', read, @@ -117,4 +117,64 @@ describe('bundleOpenApiSpecification', () => { expect(result).toEqual(expectedResult.trimStart()); }); + it('should return the bundled asyncapi specification', async () => { + const spec = ` + asyncapi: 2.5.0 + info: + version: 1.0.0 + title: Sample API + description: A sample API to illustrate OpenAPI concepts + channels: + my-topic: + subscribe: + message: + schemaFormat: "application/schema+json;version=draft-07" + payload: + $ref : "./asyncapi.schema.json" + `; + const jsonSchema = ` + { + "type": "object", + "description": "ExampleSchema", + "properties": { + "name" : { + "type": "string" + }, + "age" : { + "type" : "integer" + } + } + } + `; + const expectedSchema = ` +asyncapi: 2.5.0 +info: + version: 1.0.0 + title: Sample API + description: A sample API to illustrate OpenAPI concepts +channels: + my-topic: + subscribe: + message: + schemaFormat: application/schema+json;version=draft-07 + payload: + type: object + description: ExampleSchema + properties: + name: + type: string + age: + type: integer +`; + read.mockResolvedValue(jsonSchema); + + const result = await bundleFileWithRefs( + spec, + 'https://github.com/owner/repo/blob/main/catalog-info.yaml', + read, + resolveUrl, + ); + + expect(result).toEqual(expectedSchema.trimStart()); + }); }); diff --git a/plugins/catalog-backend-module-openapi/src/lib/bundle.ts b/plugins/catalog-backend-module-openapi/src/lib/bundle.ts index 8dd880c234..64c772b91e 100644 --- a/plugins/catalog-backend-module-openapi/src/lib/bundle.ts +++ b/plugins/catalog-backend-module-openapi/src/lib/bundle.ts @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import SwaggerParser from '@apidevtools/swagger-parser'; +import $RefParser from '@apidevtools/json-schema-ref-parser'; import { parse, stringify } from 'yaml'; import * as path from 'path'; @@ -30,13 +30,13 @@ export type BundlerRead = (url: string) => Promise; export type BundlerResolveUrl = (url: string, base: string) => string; -export async function bundleOpenApiSpecification( - specification: string, +export async function bundleFileWithRefs( + fileWithRefs: string, baseUrl: string, read: BundlerRead, resolveUrl: BundlerResolveUrl, ): Promise { - const fileUrlReaderResolver: SwaggerParser.ResolverOptions = { + const fileUrlReaderResolver: $RefParser.ResolverOptions = { canRead: file => { const protocol = getProtocol(file.url); return protocol === undefined || protocol === 'file'; @@ -47,7 +47,7 @@ export async function bundleOpenApiSpecification( return await read(url); }, }; - const httpUrlReaderResolver: SwaggerParser.ResolverOptions = { + const httpUrlReaderResolver: $RefParser.ResolverOptions = { canRead: ref => { const protocol = getProtocol(ref.url); return protocol === 'http' || protocol === 'https'; @@ -58,13 +58,13 @@ export async function bundleOpenApiSpecification( }, }; - const options: SwaggerParser.Options = { + const options: $RefParser.Options = { resolve: { file: fileUrlReaderResolver, http: httpUrlReaderResolver, }, }; - const specObject = parse(specification); - const bundledSpec = await SwaggerParser.bundle(specObject, options); - return stringify(bundledSpec); + const fileObject = parse(fileWithRefs); + const bundledObject = await $RefParser.bundle(fileObject, options); + return stringify(bundledObject); } diff --git a/plugins/catalog-backend-module-openapi/src/openApiPlaceholderResolver.test.ts b/plugins/catalog-backend-module-openapi/src/refPlaceholderResolver.test.ts similarity index 70% rename from plugins/catalog-backend-module-openapi/src/openApiPlaceholderResolver.test.ts rename to plugins/catalog-backend-module-openapi/src/refPlaceholderResolver.test.ts index f58a818ba5..9ecf0a2995 100644 --- a/plugins/catalog-backend-module-openapi/src/openApiPlaceholderResolver.test.ts +++ b/plugins/catalog-backend-module-openapi/src/refPlaceholderResolver.test.ts @@ -14,16 +14,16 @@ * limitations under the License. */ import { PlaceholderResolverParams } from '@backstage/plugin-catalog-backend'; -import { openApiPlaceholderResolver } from './openApiPlaceholderResolver'; -import { bundleOpenApiSpecification } from './lib'; +import { refPlaceholderResolver } from './refPlaceholderResolver'; +import { bundleFileWithRefs } from './lib'; jest.mock('./lib', () => ({ - bundleOpenApiSpecification: jest.fn(), + bundleFileWithRefs: jest.fn(), })); -const bundledSpecification = ''; +const bundled = ''; -describe('openApiPlaceholderResolver', () => { +describe('refPlaceholderResolver', () => { const mockResolveUrl = jest.fn(); mockResolveUrl.mockReturnValue('mockUrl'); @@ -40,7 +40,7 @@ describe('openApiPlaceholderResolver', () => { }; beforeEach(() => { - (bundleOpenApiSpecification as any).mockResolvedValue(bundledSpecification); + (bundleFileWithRefs as any).mockResolvedValue(bundled); }); afterEach(() => { @@ -48,16 +48,16 @@ describe('openApiPlaceholderResolver', () => { }); it('should throw error if unable to bundle the OpenAPI specification', async () => { - (bundleOpenApiSpecification as any).mockRejectedValue(new Error('TEST')); + (bundleFileWithRefs as any).mockRejectedValue(new Error('TEST')); - await expect(openApiPlaceholderResolver(params)).rejects.toThrow( + await expect(refPlaceholderResolver(params)).rejects.toThrow( 'Placeholder $openapi unable to bundle OpenAPI specification', ); }); it('should bundle the OpenAPI specification', async () => { - const result = await openApiPlaceholderResolver(params); + const result = await refPlaceholderResolver(params); - expect(result).toEqual(bundledSpecification); + expect(result).toEqual(bundled); }); }); diff --git a/plugins/catalog-backend-module-openapi/src/openApiPlaceholderResolver.ts b/plugins/catalog-backend-module-openapi/src/refPlaceholderResolver.ts similarity index 94% rename from plugins/catalog-backend-module-openapi/src/openApiPlaceholderResolver.ts rename to plugins/catalog-backend-module-openapi/src/refPlaceholderResolver.ts index 96fadb3cbc..39c2e639ab 100644 --- a/plugins/catalog-backend-module-openapi/src/openApiPlaceholderResolver.ts +++ b/plugins/catalog-backend-module-openapi/src/refPlaceholderResolver.ts @@ -16,10 +16,10 @@ import { PlaceholderResolverParams } from '@backstage/plugin-catalog-backend'; import { JsonValue } from '@backstage/types'; import { processingResult } from '@backstage/plugin-catalog-node'; -import { bundleOpenApiSpecification } from './lib'; +import { bundleFileWithRefs } from './lib'; /** @public */ -export async function openApiPlaceholderResolver( +export async function refPlaceholderResolver( params: PlaceholderResolverParams, ): Promise { const { content, url } = await readTextLocation(params); @@ -27,7 +27,7 @@ export async function openApiPlaceholderResolver( params.emit(processingResult.refresh(`url:${url}`)); try { - return await bundleOpenApiSpecification( + return await bundleFileWithRefs( content, url, params.read,