Adds support for generic $ref resolving in yaml/json documents.

This resolves https://github.com/backstage/backstage/issues/14490.

Signed-off-by: Thorsten Hake <mail@thorsten-hake.com>

.changeset/tender-colts-greet.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-catalog-backend-module-openapi': patch
+---
+
+Enabled support of resolving `$refs` in all kind of yaml documents, not only OpenAPI. This implicitly adds `$ref` resolving support for AsyncAPI specs. Thus, the `openApiPlaceholderResolver` has been renamed to `refPlaceholderResolver`.

plugins/catalog-backend-module-openapi/README.md

@@ -1,8 +1,10 @@
-# Catalog Backend Module for OpenAPI specifications
+# Catalog Backend Module to resolve $refs in yaml documents
 
-This is an extension module to the plugin-catalog-backend plugin, providing extensions targeted at OpenAPI specifications.
+This is an extension module to the plugin-catalog-backend plugin, providing an extensions to resolve $refs in yamls documents.
 
-With this you can split your OpenAPI definition into multiple files and reference them. They will be bundled, using an UrlReader, during processing and stored as a single specification.
+With this you can split your yaml documents into multiple files and reference them. They will be bundled, using an UrlReader, during processing and stored as a single specification.
+
+This is useful for OpenAPI and AsyncAPI specifications.
 
 ## Installation
 
@@ -15,15 +17,18 @@ yarn add --cwd packages/backend @backstage/plugin-catalog-backend-module-openapi
 
 ### Adding the plugin to your `packages/backend`
 
-#### **openApiPlaceholderResolver**
+#### **refPlaceholderResolver**
 
-The placeholder resolver can be added by importing `openApiPlaceholderResolver` in `src/plugins/catalog.ts` in your `backend` package and adding the following.
+The placeholder resolver can be added by importing `refPlaceholderResolver` in `src/plugins/catalog.ts` in your `backend` package and adding the following.
 
 ```ts
-builder.setPlaceholderResolver('openapi', openApiPlaceholderResolver);
+builder.setPlaceholderResolver('openapi', refPlaceholderResolver);
+builder.setPlaceholderResolver('asyncapi', refPlaceholderResolver);
 ```
 
-This allows you to use the `$openapi` placeholder when referencing your OpenAPI specification. This will then resolve all `$ref` instances in your specification.
+This allows you to use the `$openapi` placeholder when referencing your OpenAPI specification and `$asyncapi` when referencing your AsyncAPI specifications. This will then resolve all `$ref` instances in your specification.
+
+You can also use this resolver for other kind of yaml files to resolve $ref pointer.
 
 ```yaml
 apiVersion: backstage.io/v1alpha1

plugins/catalog-backend-module-openapi/package.json

@@ -32,7 +32,7 @@
     "start": "backstage-cli package start"
   },
   "dependencies": {
-    "@apidevtools/swagger-parser": "^10.1.0",
+    "@apidevtools/json-schema-ref-parser": "^9.0.6",
     "@backstage/backend-common": "workspace:^",
     "@backstage/catalog-model": "workspace:^",
     "@backstage/config": "workspace:^",

plugins/catalog-backend-module-openapi/src/OpenApiRefProcessor.test.ts

@@ -17,13 +17,13 @@ import { getVoidLogger } from '@backstage/backend-common';
 import { ConfigReader } from '@backstage/config';
 import { LocationSpec } from '@backstage/plugin-catalog-backend';
 import { OpenApiRefProcessor } from './OpenApiRefProcessor';
-import { bundleOpenApiSpecification } from './lib';
+import { bundleFileWithRefs } from './lib';
 
 jest.mock('./lib', () => ({
-  bundleOpenApiSpecification: jest.fn(),
+  bundleFileWithRefs: jest.fn(),
 }));
 
-const bundledSpecification = '<bundled-specification>';
+const bundled = '<bundled-specification>';
 
 describe('OpenApiRefProcessor', () => {
   const mockLocation = (): LocationSpec => ({
@@ -32,7 +32,7 @@ describe('OpenApiRefProcessor', () => {
   });
 
   beforeEach(() => {
-    (bundleOpenApiSpecification as any).mockResolvedValue(bundledSpecification);
+    (bundleFileWithRefs as any).mockResolvedValue(bundled);
   });
 
   afterEach(() => {
@@ -71,7 +71,7 @@ describe('OpenApiRefProcessor', () => {
         mockLocation(),
       );
 
-      expect(result.spec?.definition).toEqual(bundledSpecification);
+      expect(result.spec?.definition).toEqual(bundled);
     });
 
     it('should ignore other kinds', async () => {

plugins/catalog-backend-module-openapi/src/OpenApiRefProcessor.ts

@@ -21,7 +21,7 @@ import {
   CatalogProcessor,
   LocationSpec,
 } from '@backstage/plugin-catalog-backend';
-import { bundleOpenApiSpecification } from './lib';
+import { bundleFileWithRefs } from './lib';
 import { Logger } from 'winston';
 
 /**
@@ -84,7 +84,7 @@ export class OpenApiRefProcessor implements CatalogProcessor {
 
     this.logger.debug(`Bundling OpenAPI specification from ${location.target}`);
     try {
-      const bundledSpec = await bundleOpenApiSpecification(
+      const bundledSpec = await bundleFileWithRefs(
         definition.toString(),
         location.target,
         this.reader.read,

plugins/catalog-backend-module-openapi/src/index.ts

@@ -13,5 +13,12 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+import { refPlaceholderResolver } from './refPlaceholderResolver';
+
 export { OpenApiRefProcessor } from './OpenApiRefProcessor';
-export { openApiPlaceholderResolver } from './openApiPlaceholderResolver';
+export { refPlaceholderResolver } from './refPlaceholderResolver';
+/**
+ * @public
+ * @deprecated replaced by refPlaceholderResolver
+ */
+export const openApiPlaceholderResolver = refPlaceholderResolver;

plugins/catalog-backend-module-openapi/src/lib/bundle.test.ts

@@ -13,7 +13,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import { bundleOpenApiSpecification } from './bundle';
+import { bundleFileWithRefs } from './bundle';
 
 const specification = `
 openapi: "3.0.0"
@@ -70,7 +70,7 @@ paths:
                 type: string
 `;
 
-describe('bundleOpenApiSpecification', () => {
+describe('bundleFileWithRefs', () => {
   const read = jest.fn();
   const resolveUrl = jest.fn();
 
@@ -81,7 +81,7 @@ describe('bundleOpenApiSpecification', () => {
   it('should return the bundled specification', async () => {
     read.mockResolvedValue(list);
 
-    const result = await bundleOpenApiSpecification(
+    const result = await bundleFileWithRefs(
       specification,
       'https://github.com/owner/repo/blob/main/catalog-info.yaml',
       read,
@@ -108,7 +108,7 @@ describe('bundleOpenApiSpecification', () => {
 
     read.mockResolvedValue(list);
 
-    const result = await bundleOpenApiSpecification(
+    const result = await bundleFileWithRefs(
       spec,
       'https://github.com/owner/repo/blob/main/catalog-info.yaml',
       read,
@@ -117,4 +117,64 @@ describe('bundleOpenApiSpecification', () => {
 
     expect(result).toEqual(expectedResult.trimStart());
   });
+  it('should return the bundled asyncapi specification', async () => {
+    const spec = `
+      asyncapi: 2.5.0
+      info:
+        version: 1.0.0
+        title: Sample API
+        description: A sample API to illustrate OpenAPI concepts
+      channels:
+        my-topic:
+          subscribe:
+            message: 
+              schemaFormat: "application/schema+json;version=draft-07"
+              payload: 
+                $ref : "./asyncapi.schema.json"
+    `;
+    const jsonSchema = `
+      {
+        "type": "object",
+        "description": "ExampleSchema",
+        "properties": {
+          "name" : {
+            "type": "string"
+          },
+          "age" : {
+            "type" : "integer"
+          }
+        }
+      }
+    `;
+    const expectedSchema = `
+asyncapi: 2.5.0
+info:
+  version: 1.0.0
+  title: Sample API
+  description: A sample API to illustrate OpenAPI concepts
+channels:
+  my-topic:
+    subscribe:
+      message:
+        schemaFormat: application/schema+json;version=draft-07
+        payload:
+          type: object
+          description: ExampleSchema
+          properties:
+            name:
+              type: string
+            age:
+              type: integer
+`;
+    read.mockResolvedValue(jsonSchema);
+
+    const result = await bundleFileWithRefs(
+      spec,
+      'https://github.com/owner/repo/blob/main/catalog-info.yaml',
+      read,
+      resolveUrl,
+    );
+
+    expect(result).toEqual(expectedSchema.trimStart());
+  });
 });

plugins/catalog-backend-module-openapi/src/lib/bundle.ts

@@ -13,7 +13,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import SwaggerParser from '@apidevtools/swagger-parser';
+import $RefParser from '@apidevtools/json-schema-ref-parser';
 import { parse, stringify } from 'yaml';
 import * as path from 'path';
 
@@ -30,13 +30,13 @@ export type BundlerRead = (url: string) => Promise<Buffer>;
 
 export type BundlerResolveUrl = (url: string, base: string) => string;
 
-export async function bundleOpenApiSpecification(
-  specification: string,
+export async function bundleFileWithRefs(
+  fileWithRefs: string,
   baseUrl: string,
   read: BundlerRead,
   resolveUrl: BundlerResolveUrl,
 ): Promise<string> {
-  const fileUrlReaderResolver: SwaggerParser.ResolverOptions = {
+  const fileUrlReaderResolver: $RefParser.ResolverOptions = {
     canRead: file => {
       const protocol = getProtocol(file.url);
       return protocol === undefined || protocol === 'file';
@@ -47,7 +47,7 @@ export async function bundleOpenApiSpecification(
       return await read(url);
     },
   };
-  const httpUrlReaderResolver: SwaggerParser.ResolverOptions = {
+  const httpUrlReaderResolver: $RefParser.ResolverOptions = {
     canRead: ref => {
       const protocol = getProtocol(ref.url);
       return protocol === 'http' || protocol === 'https';
@@ -58,13 +58,13 @@ export async function bundleOpenApiSpecification(
     },
   };
 
-  const options: SwaggerParser.Options = {
+  const options: $RefParser.Options = {
     resolve: {
       file: fileUrlReaderResolver,
       http: httpUrlReaderResolver,
     },
   };
-  const specObject = parse(specification);
-  const bundledSpec = await SwaggerParser.bundle(specObject, options);
-  return stringify(bundledSpec);
+  const fileObject = parse(fileWithRefs);
+  const bundledObject = await $RefParser.bundle(fileObject, options);
+  return stringify(bundledObject);
 }

plugins/catalog-backend-module-openapi/src/refPlaceholderResolver.test.ts

@@ -14,16 +14,16 @@
  * limitations under the License.
  */
 import { PlaceholderResolverParams } from '@backstage/plugin-catalog-backend';
-import { openApiPlaceholderResolver } from './openApiPlaceholderResolver';
-import { bundleOpenApiSpecification } from './lib';
+import { refPlaceholderResolver } from './refPlaceholderResolver';
+import { bundleFileWithRefs } from './lib';
 
 jest.mock('./lib', () => ({
-  bundleOpenApiSpecification: jest.fn(),
+  bundleFileWithRefs: jest.fn(),
 }));
 
-const bundledSpecification = '<bundled-specification>';
+const bundled = '<bundled-specification>';
 
-describe('openApiPlaceholderResolver', () => {
+describe('refPlaceholderResolver', () => {
   const mockResolveUrl = jest.fn();
   mockResolveUrl.mockReturnValue('mockUrl');
 
@@ -40,7 +40,7 @@ describe('openApiPlaceholderResolver', () => {
   };
 
   beforeEach(() => {
-    (bundleOpenApiSpecification as any).mockResolvedValue(bundledSpecification);
+    (bundleFileWithRefs as any).mockResolvedValue(bundled);
   });
 
   afterEach(() => {
@@ -48,16 +48,16 @@ describe('openApiPlaceholderResolver', () => {
   });
 
   it('should throw error if unable to bundle the OpenAPI specification', async () => {
-    (bundleOpenApiSpecification as any).mockRejectedValue(new Error('TEST'));
+    (bundleFileWithRefs as any).mockRejectedValue(new Error('TEST'));
 
-    await expect(openApiPlaceholderResolver(params)).rejects.toThrow(
+    await expect(refPlaceholderResolver(params)).rejects.toThrow(
       'Placeholder $openapi unable to bundle OpenAPI specification',
     );
   });
 
   it('should bundle the OpenAPI specification', async () => {
-    const result = await openApiPlaceholderResolver(params);
+    const result = await refPlaceholderResolver(params);
 
-    expect(result).toEqual(bundledSpecification);
+    expect(result).toEqual(bundled);
   });
 });

plugins/catalog-backend-module-openapi/src/refPlaceholderResolver.ts

@@ -16,10 +16,10 @@
 import { PlaceholderResolverParams } from '@backstage/plugin-catalog-backend';
 import { JsonValue } from '@backstage/types';
 import { processingResult } from '@backstage/plugin-catalog-node';
-import { bundleOpenApiSpecification } from './lib';
+import { bundleFileWithRefs } from './lib';
 
 /** @public */
-export async function openApiPlaceholderResolver(
+export async function refPlaceholderResolver(
   params: PlaceholderResolverParams,
 ): Promise<JsonValue> {
   const { content, url } = await readTextLocation(params);
@@ -27,7 +27,7 @@ export async function openApiPlaceholderResolver(
   params.emit(processingResult.refresh(`url:${url}`));
 
   try {
-    return await bundleOpenApiSpecification(
+    return await bundleFileWithRefs(
       content,
       url,
       params.read,