Merge pull request #14548 from thake/feat/resolve-refs-in-yaml

Adds support for generic $ref resolving in yaml documents.

.changeset/tender-colts-greet.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-catalog-backend-module-openapi': patch
+---
+
+Enabled support of resolving `$refs` in all kind of yaml documents, not only OpenAPI. This implicitly adds `$ref` resolving support for AsyncAPI specs. Thus, the `openApiPlaceholderResolver` has been renamed to `jsonSchemaRefPlaceholderResolver`.

plugins/catalog-backend-module-openapi/README.md

@@ -1,8 +1,10 @@
-# Catalog Backend Module for OpenAPI specifications
+# Catalog Backend Module to resolve $refs in yaml documents
 
-This is an extension module to the plugin-catalog-backend plugin, providing extensions targeted at OpenAPI specifications.
+This is an extension module to the Catalog backend, providing extensions to resolve $refs in yaml documents.
 
-With this you can split your OpenAPI definition into multiple files and reference them. They will be bundled, using an UrlReader, during processing and stored as a single specification.
+With this you can split your yaml documents into multiple files and reference them. They will be bundled, using an UrlReader, during processing and stored as a single specification.
+
+This is useful for OpenAPI and AsyncAPI specifications.
 
 ## Installation
 
@@ -15,15 +17,18 @@ yarn add --cwd packages/backend @backstage/plugin-catalog-backend-module-openapi
 
 ### Adding the plugin to your `packages/backend`
 
-#### **openApiPlaceholderResolver**
+#### **jsonSchemaRefPlaceholderResolver**
 
-The placeholder resolver can be added by importing `openApiPlaceholderResolver` in `src/plugins/catalog.ts` in your `backend` package and adding the following.
+The placeholder resolver can be added by importing `jsonSchemaRefPlaceholderResolver` in `src/plugins/catalog.ts` in your `backend` package and adding the following.
 
 ```ts
-builder.setPlaceholderResolver('openapi', openApiPlaceholderResolver);
+builder.setPlaceholderResolver('openapi', jsonSchemaRefPlaceholderResolver);
+builder.setPlaceholderResolver('asyncapi', jsonSchemaRefPlaceholderResolver);
 ```
 
-This allows you to use the `$openapi` placeholder when referencing your OpenAPI specification. This will then resolve all `$ref` instances in your specification.
+This allows you to use the `$openapi` placeholder when referencing your OpenAPI specification and `$asyncapi` when referencing your AsyncAPI specifications. This will then resolve all `$ref` instances in your specification.
+
+You can also use this resolver for other kind of yaml files to resolve $ref pointer.
 
 ```yaml
 apiVersion: backstage.io/v1alpha1

plugins/catalog-backend-module-openapi/api-report.md

@@ -14,10 +14,13 @@ import { ScmIntegrations } from '@backstage/integration';
 import { UrlReader } from '@backstage/backend-common';
 
 // @public (undocumented)
-export function openApiPlaceholderResolver(
+export function jsonSchemaRefPlaceholderResolver(
   params: PlaceholderResolverParams,
 ): Promise<JsonValue>;
 
+// @public @deprecated (undocumented)
+export const openApiPlaceholderResolver: typeof jsonSchemaRefPlaceholderResolver;
+
 // @public @deprecated (undocumented)
 export class OpenApiRefProcessor implements CatalogProcessor {
   constructor(options: {

plugins/catalog-backend-module-openapi/package.json

@@ -32,7 +32,7 @@
     "start": "backstage-cli package start"
   },
   "dependencies": {
-    "@apidevtools/swagger-parser": "^10.1.0",
+    "@apidevtools/json-schema-ref-parser": "^9.0.6",
     "@backstage/backend-common": "workspace:^",
     "@backstage/catalog-model": "workspace:^",
     "@backstage/config": "workspace:^",

plugins/catalog-backend-module-openapi/src/OpenApiRefProcessor.test.ts

@@ -17,13 +17,13 @@ import { getVoidLogger } from '@backstage/backend-common';
 import { ConfigReader } from '@backstage/config';
 import { LocationSpec } from '@backstage/plugin-catalog-backend';
 import { OpenApiRefProcessor } from './OpenApiRefProcessor';
-import { bundleOpenApiSpecification } from './lib';
+import { bundleFileWithRefs } from './lib';
 
 jest.mock('./lib', () => ({
-  bundleOpenApiSpecification: jest.fn(),
+  bundleFileWithRefs: jest.fn(),
 }));
 
-const bundledSpecification = '<bundled-specification>';
+const bundled = '<bundled-specification>';
 
 describe('OpenApiRefProcessor', () => {
   const mockLocation = (): LocationSpec => ({
@@ -32,7 +32,7 @@ describe('OpenApiRefProcessor', () => {
   });
 
   beforeEach(() => {
-    (bundleOpenApiSpecification as any).mockResolvedValue(bundledSpecification);
+    (bundleFileWithRefs as any).mockResolvedValue(bundled);
   });
 
   afterEach(() => {
@@ -71,7 +71,7 @@ describe('OpenApiRefProcessor', () => {
         mockLocation(),
       );
 
-      expect(result.spec?.definition).toEqual(bundledSpecification);
+      expect(result.spec?.definition).toEqual(bundled);
     });
 
     it('should ignore other kinds', async () => {

plugins/catalog-backend-module-openapi/src/OpenApiRefProcessor.ts

@@ -21,7 +21,7 @@ import {
   CatalogProcessor,
   LocationSpec,
 } from '@backstage/plugin-catalog-backend';
-import { bundleOpenApiSpecification } from './lib';
+import { bundleFileWithRefs } from './lib';
 import { Logger } from 'winston';
 
 /**
@@ -84,7 +84,7 @@ export class OpenApiRefProcessor implements CatalogProcessor {
 
     this.logger.debug(`Bundling OpenAPI specification from ${location.target}`);
     try {
-      const bundledSpec = await bundleOpenApiSpecification(
+      const bundledSpec = await bundleFileWithRefs(
         definition.toString(),
         location.target,
         this.reader.read,

plugins/catalog-backend-module-openapi/src/index.ts

@@ -13,5 +13,12 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+import { jsonSchemaRefPlaceholderResolver } from './jsonSchemaRefPlaceholderResolver';
+
 export { OpenApiRefProcessor } from './OpenApiRefProcessor';
-export { openApiPlaceholderResolver } from './openApiPlaceholderResolver';
+export { jsonSchemaRefPlaceholderResolver } from './jsonSchemaRefPlaceholderResolver';
+/**
+ * @public
+ * @deprecated replaced by jsonSchemaRefPlaceholderResolver
+ */
+export const openApiPlaceholderResolver = jsonSchemaRefPlaceholderResolver;

plugins/catalog-backend-module-openapi/src/jsonSchemaRefPlaceholderResolver.test.ts

@@ -14,16 +14,16 @@
  * limitations under the License.
  */
 import { PlaceholderResolverParams } from '@backstage/plugin-catalog-backend';
-import { openApiPlaceholderResolver } from './openApiPlaceholderResolver';
-import { bundleOpenApiSpecification } from './lib';
+import { jsonSchemaRefPlaceholderResolver } from './jsonSchemaRefPlaceholderResolver';
+import { bundleFileWithRefs } from './lib';
 
 jest.mock('./lib', () => ({
-  bundleOpenApiSpecification: jest.fn(),
+  bundleFileWithRefs: jest.fn(),
 }));
 
-const bundledSpecification = '<bundled-specification>';
+const bundled = '<bundled-specification>';
 
-describe('openApiPlaceholderResolver', () => {
+describe('jsonSchemaRefPlaceholderResolver', () => {
   const mockResolveUrl = jest.fn();
   mockResolveUrl.mockReturnValue('mockUrl');
 
@@ -40,7 +40,7 @@ describe('openApiPlaceholderResolver', () => {
   };
 
   beforeEach(() => {
-    (bundleOpenApiSpecification as any).mockResolvedValue(bundledSpecification);
+    (bundleFileWithRefs as any).mockResolvedValue(bundled);
   });
 
   afterEach(() => {
@@ -48,16 +48,14 @@ describe('openApiPlaceholderResolver', () => {
   });
 
   it('should throw error if unable to bundle the OpenAPI specification', async () => {
-    (bundleOpenApiSpecification as any).mockRejectedValue(new Error('TEST'));
+    (bundleFileWithRefs as any).mockRejectedValue(new Error('TEST'));
 
-    await expect(openApiPlaceholderResolver(params)).rejects.toThrow(
-      'Placeholder $openapi unable to bundle OpenAPI specification',
-    );
+    await expect(jsonSchemaRefPlaceholderResolver(params)).rejects.toThrow();
   });
 
   it('should bundle the OpenAPI specification', async () => {
-    const result = await openApiPlaceholderResolver(params);
+    const result = await jsonSchemaRefPlaceholderResolver(params);
 
-    expect(result).toEqual(bundledSpecification);
+    expect(result).toEqual(bundled);
   });
 });

plugins/catalog-backend-module-openapi/src/jsonSchemaRefPlaceholderResolver.ts

@@ -16,10 +16,10 @@
 import { PlaceholderResolverParams } from '@backstage/plugin-catalog-backend';
 import { JsonValue } from '@backstage/types';
 import { processingResult } from '@backstage/plugin-catalog-node';
-import { bundleOpenApiSpecification } from './lib';
+import { bundleFileWithRefs } from './lib';
 
 /** @public */
-export async function openApiPlaceholderResolver(
+export async function jsonSchemaRefPlaceholderResolver(
   params: PlaceholderResolverParams,
 ): Promise<JsonValue> {
   const { content, url } = await readTextLocation(params);
@@ -27,7 +27,7 @@ export async function openApiPlaceholderResolver(
   params.emit(processingResult.refresh(`url:${url}`));
 
   try {
-    return await bundleOpenApiSpecification(
+    return await bundleFileWithRefs(
       content,
       url,
       params.read,
@@ -35,7 +35,7 @@ export async function openApiPlaceholderResolver(
     );
   } catch (error) {
     throw new Error(
-      `Placeholder \$${params.key} unable to bundle OpenAPI specification at ${params.value}, ${error}`,
+      `Placeholder \$${params.key} unable to bundle the file at ${params.value}, ${error}`,
     );
   }
 }

plugins/catalog-backend-module-openapi/src/lib/bundle.test.ts

@@ -13,7 +13,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import { bundleOpenApiSpecification } from './bundle';
+import { bundleFileWithRefs } from './bundle';
 
 const specification = `
 openapi: "3.0.0"
@@ -70,7 +70,7 @@ paths:
                 type: string
 `;
 
-describe('bundleOpenApiSpecification', () => {
+describe('bundleFileWithRefs', () => {
   const read = jest.fn();
   const resolveUrl = jest.fn();
 
@@ -81,7 +81,7 @@ describe('bundleOpenApiSpecification', () => {
   it('should return the bundled specification', async () => {
     read.mockResolvedValue(list);
 
-    const result = await bundleOpenApiSpecification(
+    const result = await bundleFileWithRefs(
       specification,
       'https://github.com/owner/repo/blob/main/catalog-info.yaml',
       read,
@@ -108,7 +108,7 @@ describe('bundleOpenApiSpecification', () => {
 
     read.mockResolvedValue(list);
 
-    const result = await bundleOpenApiSpecification(
+    const result = await bundleFileWithRefs(
       spec,
       'https://github.com/owner/repo/blob/main/catalog-info.yaml',
       read,
@@ -117,4 +117,64 @@ describe('bundleOpenApiSpecification', () => {
 
     expect(result).toEqual(expectedResult.trimStart());
   });
+  it('should return the bundled asyncapi specification', async () => {
+    const spec = `
+      asyncapi: 2.5.0
+      info:
+        version: 1.0.0
+        title: Sample API
+        description: A sample API to illustrate OpenAPI concepts
+      channels:
+        my-topic:
+          subscribe:
+            message: 
+              schemaFormat: "application/schema+json;version=draft-07"
+              payload: 
+                $ref : "./asyncapi.schema.json"
+    `;
+    const jsonSchema = `
+      {
+        "type": "object",
+        "description": "ExampleSchema",
+        "properties": {
+          "name" : {
+            "type": "string"
+          },
+          "age" : {
+            "type" : "integer"
+          }
+        }
+      }
+    `;
+    const expectedSchema = `
+asyncapi: 2.5.0
+info:
+  version: 1.0.0
+  title: Sample API
+  description: A sample API to illustrate OpenAPI concepts
+channels:
+  my-topic:
+    subscribe:
+      message:
+        schemaFormat: application/schema+json;version=draft-07
+        payload:
+          type: object
+          description: ExampleSchema
+          properties:
+            name:
+              type: string
+            age:
+              type: integer
+`;
+    read.mockResolvedValue(jsonSchema);
+
+    const result = await bundleFileWithRefs(
+      spec,
+      'https://github.com/owner/repo/blob/main/catalog-info.yaml',
+      read,
+      resolveUrl,
+    );
+
+    expect(result).toEqual(expectedSchema.trimStart());
+  });
 });

plugins/catalog-backend-module-openapi/src/lib/bundle.ts

@@ -13,7 +13,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import SwaggerParser from '@apidevtools/swagger-parser';
+import $RefParser from '@apidevtools/json-schema-ref-parser';
 import { parse, stringify } from 'yaml';
 import * as path from 'path';
 
@@ -30,13 +30,13 @@ export type BundlerRead = (url: string) => Promise<Buffer>;
 
 export type BundlerResolveUrl = (url: string, base: string) => string;
 
-export async function bundleOpenApiSpecification(
-  specification: string,
+export async function bundleFileWithRefs(
+  fileWithRefs: string,
   baseUrl: string,
   read: BundlerRead,
   resolveUrl: BundlerResolveUrl,
 ): Promise<string> {
-  const fileUrlReaderResolver: SwaggerParser.ResolverOptions = {
+  const fileUrlReaderResolver: $RefParser.ResolverOptions = {
     canRead: file => {
       const protocol = getProtocol(file.url);
       return protocol === undefined || protocol === 'file';
@@ -47,7 +47,7 @@ export async function bundleOpenApiSpecification(
       return await read(url);
     },
   };
-  const httpUrlReaderResolver: SwaggerParser.ResolverOptions = {
+  const httpUrlReaderResolver: $RefParser.ResolverOptions = {
     canRead: ref => {
       const protocol = getProtocol(ref.url);
       return protocol === 'http' || protocol === 'https';
@@ -58,13 +58,13 @@ export async function bundleOpenApiSpecification(
     },
   };
 
-  const options: SwaggerParser.Options = {
+  const options: $RefParser.Options = {
     resolve: {
       file: fileUrlReaderResolver,
       http: httpUrlReaderResolver,
     },
   };
-  const specObject = parse(specification);
-  const bundledSpec = await SwaggerParser.bundle(specObject, options);
-  return stringify(bundledSpec);
+  const fileObject = parse(fileWithRefs);
+  const bundledObject = await $RefParser.bundle(fileObject, options);
+  return stringify(bundledObject);
 }

yarn.lock

@@ -21,7 +21,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@apidevtools/json-schema-ref-parser@npm:9.0.6, @apidevtools/json-schema-ref-parser@npm:^9.0.6":
+"@apidevtools/json-schema-ref-parser@npm:^9.0.6":
   version: 9.0.6
   resolution: "@apidevtools/json-schema-ref-parser@npm:9.0.6"
   dependencies:
@@ -32,37 +32,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@apidevtools/openapi-schemas@npm:^2.1.0":
-  version: 2.1.0
-  resolution: "@apidevtools/openapi-schemas@npm:2.1.0"
-  checksum: 4a8f64935b9049ef21e41fa4b188f39f6bc3f5291cebd451701db1115451ccb246a739e46cc5ce9ecdec781671431db40db7851acdac84a990a45756e0f32de3
-  languageName: node
-  linkType: hard
-
-"@apidevtools/swagger-methods@npm:^3.0.2":
-  version: 3.0.2
-  resolution: "@apidevtools/swagger-methods@npm:3.0.2"
-  checksum: d06b1ac5c1956613c4c6be695612ef860cd4e962b93a509ca551735a328a856cae1e33399cac1dcbf8333ba22b231746f3586074769ef0e172cf549ec9e7eaae
-  languageName: node
-  linkType: hard
-
-"@apidevtools/swagger-parser@npm:^10.1.0":
-  version: 10.1.0
-  resolution: "@apidevtools/swagger-parser@npm:10.1.0"
-  dependencies:
-    "@apidevtools/json-schema-ref-parser": 9.0.6
-    "@apidevtools/openapi-schemas": ^2.1.0
-    "@apidevtools/swagger-methods": ^3.0.2
-    "@jsdevtools/ono": ^7.1.3
-    ajv: ^8.6.3
-    ajv-draft-04: ^1.0.0
-    call-me-maybe: ^1.0.1
-  peerDependencies:
-    openapi-types: ">=7"
-  checksum: c7c923755bd025ee2cae97e1cfd525538523ba74c341a0ac814c023ffe5e63fc2d997539a8ccf9a0fcec41a2d6337d40cc5735acb991ddcbb415853a241908d1
-  languageName: node
-  linkType: hard
-
 "@apollo/explorer@npm:^1.1.1":
   version: 1.2.0
   resolution: "@apollo/explorer@npm:1.2.0"
@@ -4701,7 +4670,7 @@ __metadata:
   version: 0.0.0-use.local
   resolution: "@backstage/plugin-catalog-backend-module-openapi@workspace:plugins/catalog-backend-module-openapi"
   dependencies:
-    "@apidevtools/swagger-parser": ^10.1.0
+    "@apidevtools/json-schema-ref-parser": ^9.0.6
     "@backstage/backend-common": "workspace:^"
     "@backstage/backend-test-utils": "workspace:^"
     "@backstage/catalog-model": "workspace:^"
@@ -15434,18 +15403,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"ajv-draft-04@npm:^1.0.0":
-  version: 1.0.0
-  resolution: "ajv-draft-04@npm:1.0.0"
-  peerDependencies:
-    ajv: ^8.5.0
-  peerDependenciesMeta:
-    ajv:
-      optional: true
-  checksum: 3f11fa0e7f7359bef6608657f02ab78e9cc62b1fb7bdd860db0d00351b3863a1189c1a23b72466d2d82726cab4eb20725c76f5e7c134a89865e2bfd0e6828137
-  languageName: node
-  linkType: hard
-
 "ajv-formats@npm:^2.1.1":
   version: 2.1.1
   resolution: "ajv-formats@npm:2.1.1"
@@ -15480,7 +15437,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"ajv8@npm:ajv@^8.11.0, ajv@npm:^8.0.0, ajv@npm:^8.10.0, ajv@npm:^8.6.3, ajv@npm:^8.8.0":
+"ajv8@npm:ajv@^8.11.0, ajv@npm:^8.0.0, ajv@npm:^8.10.0, ajv@npm:^8.8.0":
   version: 8.11.2
   resolution: "ajv@npm:8.11.2"
   dependencies: