Merge pull request #11645 from mfrinnstrom/add-openapi-multifile-support

Add OpenApiRefProcessor for handling $ref in OpenAPI specs
2022-06-23 11:42:28 +02:00
parent 67fe99a656 7d9dc8f271
commit b0771b4976
13 changed files with 589 additions and 2 deletions
@@ -0,0 +1,7 @@
+---
+'@backstage/plugin-catalog-backend-module-openapi': minor
+---
+
+Add basic OpenAPI \$ref support.
+
+For more information see [here](https://github.com/backstage/backstage/tree/master/plugins/catalog-backend-module-openapi).
@@ -0,0 +1 @@
+module.exports = require('@backstage/cli/config/eslint-factory')(__dirname);
@@ -0,0 +1,27 @@
+# Catalog Backend Module for OpenAPI specifications
+
+This is an extension module to the plugin-catalog-backend plugin, providing extensions targeted at OpenAPI specifications.
+
+With this you can split your OpenAPI definition into multiple files and reference them. They will be bundled, using an UrlReader, during processing and stored as a single specification.
+
+## Installation
+
+### Install the package
+
+```bash
+# From your Backstage root directory
+yarn add --cwd packages/backend @backstage/plugin-catalog-backend-module-openapi
+```
+
+### Adding the plugin to your `packages/backend`
+
+The processor can be added by importing `OpenApiRefProcessor` in `src/plugins/catalog.ts` in your `backend` package and adding the following.
+
+```ts
+builder.addProcessor(
+  OpenApiRefProcessor.fromConfig(env.config, {
+    logger: env.logger,
+    reader: env.reader,
+  }),
+);
+```
@@ -0,0 +1,36 @@
+## API Report File for "@backstage/plugin-catalog-backend-module-openapi"
+
+> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
+
+```ts
+import { CatalogProcessor } from '@backstage/plugin-catalog-backend';
+import { Config } from '@backstage/config';
+import { Entity } from '@backstage/catalog-model';
+import { LocationSpec } from '@backstage/plugin-catalog-backend';
+import { Logger } from 'winston';
+import { ScmIntegrations } from '@backstage/integration';
+import { UrlReader } from '@backstage/backend-common';
+
+// @public (undocumented)
+export class OpenApiRefProcessor implements CatalogProcessor {
+  constructor(options: {
+    integrations: ScmIntegrations;
+    logger: Logger;
+    reader: UrlReader;
+  });
+  // (undocumented)
+  static fromConfig(
+    config: Config,
+    options: {
+      logger: Logger;
+      reader: UrlReader;
+    },
+  ): OpenApiRefProcessor;
+  // (undocumented)
+  getProcessorName(): string;
+  // (undocumented)
+  preProcessEntity(entity: Entity, location: LocationSpec): Promise<Entity>;
+}
+
+// (No @packageDocumentation comment for this package)
+```
@@ -0,0 +1,53 @@
+{
+  "name": "@backstage/plugin-catalog-backend-module-openapi",
+  "description": "A Backstage catalog backend module that helps with OpenAPI specifications",
+  "version": "0.0.0",
+  "main": "src/index.ts",
+  "types": "src/index.ts",
+  "license": "Apache-2.0",
+  "private": false,
+  "publishConfig": {
+    "access": "public",
+    "main": "dist/index.cjs.js",
+    "types": "dist/index.d.ts"
+  },
+  "backstage": {
+    "role": "backend-plugin-module"
+  },
+  "homepage": "https://backstage.io",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/backstage/backstage",
+    "directory": "plugins/catalog-backend-module-openapi"
+  },
+  "keywords": [
+    "backstage"
+  ],
+  "scripts": {
+    "build": "backstage-cli package build",
+    "lint": "backstage-cli package lint",
+    "test": "backstage-cli package test",
+    "prepack": "backstage-cli package prepack",
+    "postpack": "backstage-cli package postpack",
+    "clean": "backstage-cli package clean",
+    "start": "backstage-cli package start"
+  },
+  "dependencies": {
+    "@apidevtools/swagger-parser": "^10.1.0",
+    "@backstage/backend-common": "^0.14.1-next.0",
+    "@backstage/catalog-model": "^1.1.0-next.0",
+    "@backstage/config": "^1.0.1",
+    "@backstage/integration": "^1.2.2-next.0",
+    "@backstage/plugin-catalog-backend": "^1.2.1-next.0",
+    "winston": "^3.2.1",
+    "yaml": "^2.1.1"
+  },
+  "devDependencies": {
+    "@backstage/backend-test-utils": "^0.1.26-next.0",
+    "@backstage/cli": "^0.17.3-next.0",
+    "openapi-types": "^11.0.1"
+  },
+  "files": [
+    "dist"
+  ]
+}
@@ -0,0 +1,101 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { getVoidLogger } from '@backstage/backend-common';
+import { ConfigReader } from '@backstage/config';
+import { LocationSpec } from '@backstage/plugin-catalog-backend';
+import { OpenApiRefProcessor } from './OpenApiRefProcessor';
+import { bundleOpenApiSpecification } from './lib';
+
+jest.mock('./lib', () => ({
+  bundleOpenApiSpecification: jest.fn(),
+}));
+
+const bundledSpecification = '<bundled-specification>';
+
+describe('OpenApiRefProcessor', () => {
+  const mockLocation = (): LocationSpec => ({
+    type: 'url',
+    target: `https://github.com/owner/repo/blob/main/catalog-info.yaml`,
+  });
+
+  beforeEach(() => {
+    (bundleOpenApiSpecification as any).mockResolvedValue(bundledSpecification);
+  });
+
+  afterEach(() => {
+    jest.clearAllMocks();
+  });
+
+  describe('preProcessEntity', () => {
+    const setupTest = ({ kind = 'API', spec = {} } = {}) => {
+      const entity = {
+        kind,
+        spec: { definition: '<openapi-definition>', ...spec },
+      };
+      const config = new ConfigReader({});
+      const reader = {
+        read: jest.fn(),
+        readTree: jest.fn(),
+        search: jest.fn(),
+      };
+      const processor = OpenApiRefProcessor.fromConfig(config, {
+        logger: getVoidLogger(),
+        reader,
+      });
+
+      return { entity, processor };
+    };
+
+    it('should bundle OpenAPI specifications', async () => {
+      const { entity, processor } = setupTest({
+        kind: 'API',
+        spec: { type: 'openapi' },
+      });
+
+      const result = await processor.preProcessEntity(
+        entity as any,
+        mockLocation(),
+      );
+
+      expect(result.spec?.definition).toEqual(bundledSpecification);
+    });
+
+    it('should ignore other kinds', async () => {
+      const { entity, processor } = setupTest({ kind: 'Group' });
+
+      const result = await processor.preProcessEntity(
+        entity as any,
+        mockLocation(),
+      );
+
+      expect(result).toEqual(entity);
+    });
+
+    it('should ignore other specification types', async () => {
+      const { entity, processor } = setupTest({
+        kind: 'Group',
+        spec: { type: 'asyncapi' },
+      });
+
+      const result = await processor.preProcessEntity(
+        entity as any,
+        mockLocation(),
+      );
+
+      expect(result).toEqual(entity);
+    });
+  });
+});
@@ -0,0 +1,94 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { UrlReader } from '@backstage/backend-common';
+import { Entity } from '@backstage/catalog-model';
+import { Config } from '@backstage/config';
+import { ScmIntegrations } from '@backstage/integration';
+import {
+  CatalogProcessor,
+  LocationSpec,
+} from '@backstage/plugin-catalog-backend';
+import { bundleOpenApiSpecification } from './lib';
+import { Logger } from 'winston';
+
+/** @public */
+export class OpenApiRefProcessor implements CatalogProcessor {
+  private readonly integrations: ScmIntegrations;
+  private readonly logger: Logger;
+  private readonly reader: UrlReader;
+
+  static fromConfig(
+    config: Config,
+    options: { logger: Logger; reader: UrlReader },
+  ) {
+    const integrations = ScmIntegrations.fromConfig(config);
+
+    return new OpenApiRefProcessor({
+      ...options,
+      integrations,
+    });
+  }
+
+  constructor(options: {
+    integrations: ScmIntegrations;
+    logger: Logger;
+    reader: UrlReader;
+  }) {
+    this.integrations = options.integrations;
+    this.logger = options.logger;
+    this.reader = options.reader;
+  }
+
+  getProcessorName(): string {
+    return 'OpenApiRefProcessor';
+  }
+
+  async preProcessEntity(
+    entity: Entity,
+    location: LocationSpec,
+  ): Promise<Entity> {
+    if (
+      !entity ||
+      entity.kind !== 'API' ||
+      (entity.spec && entity.spec.type !== 'openapi')
+    ) {
+      return entity;
+    }
+
+    const scmIntegration = this.integrations.byUrl(location.target);
+    if (!scmIntegration) {
+      return entity;
+    }
+
+    this.logger.debug(`Bundling OpenAPI specification from ${location.target}`);
+    try {
+      const bundledSpec = await bundleOpenApiSpecification(
+        entity.spec!.definition?.toString(),
+        location.target,
+        this.reader,
+        scmIntegration,
+      );
+
+      return {
+        ...entity,
+        spec: { ...entity.spec, definition: bundledSpec },
+      };
+    } catch (error) {
+      this.logger.error(`Unable to bundle OpenAPI specification`, error);
+      return entity;
+    }
+  }
+}
@@ -0,0 +1,16 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+export { OpenApiRefProcessor } from './OpenApiRefProcessor';
@@ -0,0 +1,117 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { ConfigReader } from '@backstage/config';
+import { ScmIntegrations } from '@backstage/integration';
+import { bundleOpenApiSpecification } from './bundle';
+
+const specification = `
+openapi: "3.0.0"
+info:
+  version: 1.0.0
+  title: Swagger Petstore
+  license:
+    name: MIT
+servers:
+  - url: http://petstore.swagger.io/v1
+paths:
+  /pets:
+    get:
+      $ref: "./paths/pets/list.yaml"
+`;
+
+const list = `
+---
+summary: List all pets
+operationId: listPets
+tags:
+  - pets
+responses:
+  '200':
+    description: A paged array of pets
+    content:
+      application/json:    
+        schema:
+          type: string
+`;
+
+const expectedResult = `
+openapi: 3.0.0
+info:
+  version: 1.0.0
+  title: Swagger Petstore
+  license:
+    name: MIT
+servers:
+  - url: http://petstore.swagger.io/v1
+paths:
+  /pets:
+    get:
+      summary: List all pets
+      operationId: listPets
+      tags:
+        - pets
+      responses:
+        "200":
+          description: A paged array of pets
+          content:
+            application/json:
+              schema:
+                type: string
+`;
+
+describe('bundleOpenApiSpecification', () => {
+  const readUrl = jest.fn();
+  const reader = {
+    readUrl,
+    read: jest.fn(),
+    readTree: jest.fn(),
+    search: jest.fn(),
+  };
+
+  const scmIntegration = ScmIntegrations.fromConfig(new ConfigReader({})).byUrl(
+    'https://github.com/owner/repo/blob/main/openapi.yaml',
+  );
+
+  beforeEach(() => {
+    jest.clearAllMocks();
+  });
+
+  it('should return undefined if no specification is supplied', async () => {
+    expect(
+      await bundleOpenApiSpecification(
+        undefined,
+        'https://github.com/owner/repo/blob/main/openapi.yaml',
+        reader,
+        scmIntegration as any,
+      ),
+    ).toBeUndefined();
+  });
+
+  it('should return the bundled specification', async () => {
+    readUrl.mockResolvedValue({
+      buffer: jest.fn().mockResolvedValue(list),
+    });
+
+    const result = await bundleOpenApiSpecification(
+      specification,
+      'https://github.com/owner/repo/blob/main/openapi.yaml',
+      reader,
+      scmIntegration as any,
+    );
+
+    expect(result).toEqual(expectedResult.trimStart());
+  });
+});
@@ -0,0 +1,69 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { UrlReader } from '@backstage/backend-common';
+import { ScmIntegration } from '@backstage/integration';
+import SwaggerParser from '@apidevtools/swagger-parser';
+import { parse, stringify } from 'yaml';
+import * as path from 'path';
+
+const protocolPattern = /^(\w{2,}):\/\//i;
+const getProtocol = (refPath: string) => {
+  const match = protocolPattern.exec(refPath);
+  if (match) {
+    return match[1].toLowerCase();
+  }
+  return undefined;
+};
+
+export async function bundleOpenApiSpecification(
+  specification: string | undefined,
+  targetUrl: string,
+  reader: UrlReader,
+  scmIntegration: ScmIntegration,
+): Promise<string | undefined> {
+  const fileUrlReaderResolver: SwaggerParser.ResolverOptions = {
+    canRead: file => {
+      const protocol = getProtocol(file.url);
+      return protocol === undefined || protocol === 'file';
+    },
+    read: async file => {
+      const relativePath = path.relative('.', file.url);
+      const url = scmIntegration.resolveUrl({
+        base: targetUrl,
+        url: relativePath,
+      });
+      if (reader.readUrl) {
+        const data = await reader.readUrl(url);
+        return data.buffer();
+      }
+      throw new Error('UrlReader has no readUrl method defined');
+    },
+  };
+
+  if (!specification) {
+    return undefined;
+  }
+
+  const options: SwaggerParser.Options = {
+    resolve: {
+      file: fileUrlReaderResolver,
+      http: true,
+    },
+  };
+  const specObject = parse(specification);
+  const bundledSpec = await SwaggerParser.bundle(specObject, options);
+  return stringify(bundledSpec);
+}
@@ -0,0 +1,16 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+export * from './bundle';
@@ -0,0 +1,17 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export {};
@@ -16,7 +16,7 @@
  dependencies:
    "@jridgewell/trace-mapping" "^0.3.0"

-"@apidevtools/json-schema-ref-parser@^9.0.6":
+"@apidevtools/json-schema-ref-parser@9.0.6", "@apidevtools/json-schema-ref-parser@^9.0.6":
  version "9.0.6"
  resolved "https://registry.npmjs.org/@apidevtools/json-schema-ref-parser/-/json-schema-ref-parser-9.0.6.tgz#5d9000a3ac1fd25404da886da6b266adcd99cf1c"
  integrity sha512-M3YgsLjI0lZxvrpeGVk9Ap032W6TPQkH6pRAZz81Ac3WUNF79VQooAFnp8umjvVzUmD93NkogxEwbSce7qMsUg==
@@ -25,6 +25,29 @@
    call-me-maybe "^1.0.1"
    js-yaml "^3.13.1"

+"@apidevtools/openapi-schemas@^2.1.0":
+  version "2.1.0"
+  resolved "https://registry.npmjs.org/@apidevtools/openapi-schemas/-/openapi-schemas-2.1.0.tgz#9fa08017fb59d80538812f03fc7cac5992caaa17"
+  integrity sha512-Zc1AlqrJlX3SlpupFGpiLi2EbteyP7fXmUOGup6/DnkRgjP9bgMM/ag+n91rsv0U1Gpz0H3VILA/o3bW7Ua6BQ==
+
+"@apidevtools/swagger-methods@^3.0.2":
+  version "3.0.2"
+  resolved "https://registry.npmjs.org/@apidevtools/swagger-methods/-/swagger-methods-3.0.2.tgz#b789a362e055b0340d04712eafe7027ddc1ac267"
+  integrity sha512-QAkD5kK2b1WfjDS/UQn/qQkbwF31uqRjPTrsCs5ZG9BQGAkjwvqGFjjPqAuzac/IYzpPtRzjCP1WrTuAIjMrXg==
+
+"@apidevtools/swagger-parser@^10.1.0":
+  version "10.1.0"
+  resolved "https://registry.npmjs.org/@apidevtools/swagger-parser/-/swagger-parser-10.1.0.tgz#a987d71e5be61feb623203be0c96e5985b192ab6"
+  integrity sha512-9Kt7EuS/7WbMAUv2gSziqjvxwDbFSg3Xeyfuj5laUODX8o/k/CpsAKiQ8W7/R88eXFTMbJYg6+7uAmOWNKmwnw==
+  dependencies:
+    "@apidevtools/json-schema-ref-parser" "9.0.6"
+    "@apidevtools/openapi-schemas" "^2.1.0"
+    "@apidevtools/swagger-methods" "^3.0.2"
+    "@jsdevtools/ono" "^7.1.3"
+    ajv "^8.6.3"
+    ajv-draft-04 "^1.0.0"
+    call-me-maybe "^1.0.1"
+
 "@apollo/protobufjs@1.2.2":
  version "1.2.2"
  resolved "https://registry.npmjs.org/@apollo/protobufjs/-/protobufjs-1.2.2.tgz#4bd92cd7701ccaef6d517cdb75af2755f049f87c"
@@ -7613,6 +7636,11 @@ aggregate-error@^3.0.0, aggregate-error@^3.1.0:
    clean-stack "^2.0.0"
    indent-string "^4.0.0"

+ajv-draft-04@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.npmjs.org/ajv-draft-04/-/ajv-draft-04-1.0.0.tgz#3b64761b268ba0b9e668f0b41ba53fce0ad77fc8"
+  integrity sha512-mv00Te6nmYbRp5DCwclxtt7yV/joXJPGS7nM+97GdxvuttCOfgI3K4U25zboyeX0O+myI8ERluxQe5wljMmVIw==
+
 ajv-formats@^2.1.1:
  version "2.1.1"
  resolved "https://registry.npmjs.org/ajv-formats/-/ajv-formats-2.1.1.tgz#6e669400659eb74973bbf2e33327180a0996b520"
@@ -7642,7 +7670,7 @@ ajv@^6.10.0, ajv@^6.10.1, ajv@^6.12.2, ajv@^6.12.4, ajv@^6.12.5, ajv@^6.5.5, ajv
    json-schema-traverse "^0.4.1"
    uri-js "^4.2.2"

-ajv@^8.0.0, ajv@^8.10.0, ajv@^8.8.0:
+ajv@^8.0.0, ajv@^8.10.0, ajv@^8.6.3, ajv@^8.8.0:
  version "8.11.0"
  resolved "https://registry.npmjs.org/ajv/-/ajv-8.11.0.tgz#977e91dd96ca669f54a11e23e378e33b884a565f"
  integrity sha512-wGgprdCvMalC0BztXvitD2hC04YffAvtsUn93JbGXYLAtCUO4xd17mCCZQxUOItiBwZvJScWo8NIvQMQ71rdpg==
@@ -19627,6 +19655,11 @@ openapi-sampler@^1.2.1:
    "@types/json-schema" "^7.0.7"
    json-pointer "0.6.2"

+openapi-types@^11.0.1:
+  version "11.1.0"
+  resolved "https://registry.npmjs.org/openapi-types/-/openapi-types-11.1.0.tgz#037969f3dfa5999423ee33bf889fb0d12984277e"
+  integrity sha512-ZW+Jf12flFF6DXSij8DGL3svDA4RtSyHXjC/xB/JAh18gg3uVfVIFLvCfScUMowrpvlkxsMMbErakbth2g3/iQ==
+
 openid-client@^4.1.1:
  version "4.9.0"
  resolved "https://registry.npmjs.org/openid-client/-/openid-client-4.9.0.tgz#bdfc9194435316df419f759ce177635146b43074"
				`@@ -0,0 +1 @@`
				`module.exports = require('@backstage/cli/config/eslint-factory')(__dirname);`