diff --git a/.changeset/shy-cameras-develop.md b/.changeset/shy-cameras-develop.md new file mode 100644 index 0000000000..c62a9a56e3 --- /dev/null +++ b/.changeset/shy-cameras-develop.md @@ -0,0 +1,7 @@ +--- +'@backstage/plugin-catalog-backend-module-openapi': minor +--- + +Add basic OpenAPI \$ref support. + +For more information see [here](https://github.com/backstage/backstage/tree/master/plugins/catalog-backend-module-openapi). diff --git a/plugins/catalog-backend-module-openapi/.eslintrc.js b/plugins/catalog-backend-module-openapi/.eslintrc.js new file mode 100644 index 0000000000..e2a53a6ad2 --- /dev/null +++ b/plugins/catalog-backend-module-openapi/.eslintrc.js @@ -0,0 +1 @@ +module.exports = require('@backstage/cli/config/eslint-factory')(__dirname); diff --git a/plugins/catalog-backend-module-openapi/README.md b/plugins/catalog-backend-module-openapi/README.md new file mode 100644 index 0000000000..1a133521c6 --- /dev/null +++ b/plugins/catalog-backend-module-openapi/README.md @@ -0,0 +1,27 @@ +# Catalog Backend Module for OpenAPI specifications + +This is an extension module to the plugin-catalog-backend plugin, providing extensions targeted at OpenAPI specifications. + +With this you can split your OpenAPI definition into multiple files and reference them. They will be bundled, using an UrlReader, during processing and stored as a single specification. + +## Installation + +### Install the package + +```bash +# From your Backstage root directory +yarn add --cwd packages/backend @backstage/plugin-catalog-backend-module-openapi +``` + +### Adding the plugin to your `packages/backend` + +The processor can be added by importing `OpenApiRefProcessor` in `src/plugins/catalog.ts` in your `backend` package and adding the following. + +```ts +builder.addProcessor( + OpenApiRefProcessor.fromConfig(env.config, { + logger: env.logger, + reader: env.reader, + }), +); +``` diff --git a/plugins/catalog-backend-module-openapi/api-report.md b/plugins/catalog-backend-module-openapi/api-report.md new file mode 100644 index 0000000000..92ddd1e3cd --- /dev/null +++ b/plugins/catalog-backend-module-openapi/api-report.md @@ -0,0 +1,36 @@ +## API Report File for "@backstage/plugin-catalog-backend-module-openapi" + +> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). + +```ts +import { CatalogProcessor } from '@backstage/plugin-catalog-backend'; +import { Config } from '@backstage/config'; +import { Entity } from '@backstage/catalog-model'; +import { LocationSpec } from '@backstage/plugin-catalog-backend'; +import { Logger } from 'winston'; +import { ScmIntegrations } from '@backstage/integration'; +import { UrlReader } from '@backstage/backend-common'; + +// @public (undocumented) +export class OpenApiRefProcessor implements CatalogProcessor { + constructor(options: { + integrations: ScmIntegrations; + logger: Logger; + reader: UrlReader; + }); + // (undocumented) + static fromConfig( + config: Config, + options: { + logger: Logger; + reader: UrlReader; + }, + ): OpenApiRefProcessor; + // (undocumented) + getProcessorName(): string; + // (undocumented) + preProcessEntity(entity: Entity, location: LocationSpec): Promise; +} + +// (No @packageDocumentation comment for this package) +``` diff --git a/plugins/catalog-backend-module-openapi/package.json b/plugins/catalog-backend-module-openapi/package.json new file mode 100644 index 0000000000..1c99618ff8 --- /dev/null +++ b/plugins/catalog-backend-module-openapi/package.json @@ -0,0 +1,53 @@ +{ + "name": "@backstage/plugin-catalog-backend-module-openapi", + "description": "A Backstage catalog backend module that helps with OpenAPI specifications", + "version": "0.0.0", + "main": "src/index.ts", + "types": "src/index.ts", + "license": "Apache-2.0", + "private": false, + "publishConfig": { + "access": "public", + "main": "dist/index.cjs.js", + "types": "dist/index.d.ts" + }, + "backstage": { + "role": "backend-plugin-module" + }, + "homepage": "https://backstage.io", + "repository": { + "type": "git", + "url": "https://github.com/backstage/backstage", + "directory": "plugins/catalog-backend-module-openapi" + }, + "keywords": [ + "backstage" + ], + "scripts": { + "build": "backstage-cli package build", + "lint": "backstage-cli package lint", + "test": "backstage-cli package test", + "prepack": "backstage-cli package prepack", + "postpack": "backstage-cli package postpack", + "clean": "backstage-cli package clean", + "start": "backstage-cli package start" + }, + "dependencies": { + "@apidevtools/swagger-parser": "^10.1.0", + "@backstage/backend-common": "^0.14.1-next.0", + "@backstage/catalog-model": "^1.1.0-next.0", + "@backstage/config": "^1.0.1", + "@backstage/integration": "^1.2.2-next.0", + "@backstage/plugin-catalog-backend": "^1.2.1-next.0", + "winston": "^3.2.1", + "yaml": "^2.1.1" + }, + "devDependencies": { + "@backstage/backend-test-utils": "^0.1.26-next.0", + "@backstage/cli": "^0.17.3-next.0", + "openapi-types": "^11.0.1" + }, + "files": [ + "dist" + ] +} diff --git a/plugins/catalog-backend-module-openapi/src/OpenApiRefProcessor.test.ts b/plugins/catalog-backend-module-openapi/src/OpenApiRefProcessor.test.ts new file mode 100644 index 0000000000..8f2cc571c2 --- /dev/null +++ b/plugins/catalog-backend-module-openapi/src/OpenApiRefProcessor.test.ts @@ -0,0 +1,101 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { getVoidLogger } from '@backstage/backend-common'; +import { ConfigReader } from '@backstage/config'; +import { LocationSpec } from '@backstage/plugin-catalog-backend'; +import { OpenApiRefProcessor } from './OpenApiRefProcessor'; +import { bundleOpenApiSpecification } from './lib'; + +jest.mock('./lib', () => ({ + bundleOpenApiSpecification: jest.fn(), +})); + +const bundledSpecification = ''; + +describe('OpenApiRefProcessor', () => { + const mockLocation = (): LocationSpec => ({ + type: 'url', + target: `https://github.com/owner/repo/blob/main/catalog-info.yaml`, + }); + + beforeEach(() => { + (bundleOpenApiSpecification as any).mockResolvedValue(bundledSpecification); + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + describe('preProcessEntity', () => { + const setupTest = ({ kind = 'API', spec = {} } = {}) => { + const entity = { + kind, + spec: { definition: '', ...spec }, + }; + const config = new ConfigReader({}); + const reader = { + read: jest.fn(), + readTree: jest.fn(), + search: jest.fn(), + }; + const processor = OpenApiRefProcessor.fromConfig(config, { + logger: getVoidLogger(), + reader, + }); + + return { entity, processor }; + }; + + it('should bundle OpenAPI specifications', async () => { + const { entity, processor } = setupTest({ + kind: 'API', + spec: { type: 'openapi' }, + }); + + const result = await processor.preProcessEntity( + entity as any, + mockLocation(), + ); + + expect(result.spec?.definition).toEqual(bundledSpecification); + }); + + it('should ignore other kinds', async () => { + const { entity, processor } = setupTest({ kind: 'Group' }); + + const result = await processor.preProcessEntity( + entity as any, + mockLocation(), + ); + + expect(result).toEqual(entity); + }); + + it('should ignore other specification types', async () => { + const { entity, processor } = setupTest({ + kind: 'Group', + spec: { type: 'asyncapi' }, + }); + + const result = await processor.preProcessEntity( + entity as any, + mockLocation(), + ); + + expect(result).toEqual(entity); + }); + }); +}); diff --git a/plugins/catalog-backend-module-openapi/src/OpenApiRefProcessor.ts b/plugins/catalog-backend-module-openapi/src/OpenApiRefProcessor.ts new file mode 100644 index 0000000000..01875e0779 --- /dev/null +++ b/plugins/catalog-backend-module-openapi/src/OpenApiRefProcessor.ts @@ -0,0 +1,94 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { UrlReader } from '@backstage/backend-common'; +import { Entity } from '@backstage/catalog-model'; +import { Config } from '@backstage/config'; +import { ScmIntegrations } from '@backstage/integration'; +import { + CatalogProcessor, + LocationSpec, +} from '@backstage/plugin-catalog-backend'; +import { bundleOpenApiSpecification } from './lib'; +import { Logger } from 'winston'; + +/** @public */ +export class OpenApiRefProcessor implements CatalogProcessor { + private readonly integrations: ScmIntegrations; + private readonly logger: Logger; + private readonly reader: UrlReader; + + static fromConfig( + config: Config, + options: { logger: Logger; reader: UrlReader }, + ) { + const integrations = ScmIntegrations.fromConfig(config); + + return new OpenApiRefProcessor({ + ...options, + integrations, + }); + } + + constructor(options: { + integrations: ScmIntegrations; + logger: Logger; + reader: UrlReader; + }) { + this.integrations = options.integrations; + this.logger = options.logger; + this.reader = options.reader; + } + + getProcessorName(): string { + return 'OpenApiRefProcessor'; + } + + async preProcessEntity( + entity: Entity, + location: LocationSpec, + ): Promise { + if ( + !entity || + entity.kind !== 'API' || + (entity.spec && entity.spec.type !== 'openapi') + ) { + return entity; + } + + const scmIntegration = this.integrations.byUrl(location.target); + if (!scmIntegration) { + return entity; + } + + this.logger.debug(`Bundling OpenAPI specification from ${location.target}`); + try { + const bundledSpec = await bundleOpenApiSpecification( + entity.spec!.definition?.toString(), + location.target, + this.reader, + scmIntegration, + ); + + return { + ...entity, + spec: { ...entity.spec, definition: bundledSpec }, + }; + } catch (error) { + this.logger.error(`Unable to bundle OpenAPI specification`, error); + return entity; + } + } +} diff --git a/plugins/catalog-backend-module-openapi/src/index.ts b/plugins/catalog-backend-module-openapi/src/index.ts new file mode 100644 index 0000000000..f98bee6bac --- /dev/null +++ b/plugins/catalog-backend-module-openapi/src/index.ts @@ -0,0 +1,16 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +export { OpenApiRefProcessor } from './OpenApiRefProcessor'; diff --git a/plugins/catalog-backend-module-openapi/src/lib/bundle.test.ts b/plugins/catalog-backend-module-openapi/src/lib/bundle.test.ts new file mode 100644 index 0000000000..4505f158b5 --- /dev/null +++ b/plugins/catalog-backend-module-openapi/src/lib/bundle.test.ts @@ -0,0 +1,117 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { ConfigReader } from '@backstage/config'; +import { ScmIntegrations } from '@backstage/integration'; +import { bundleOpenApiSpecification } from './bundle'; + +const specification = ` +openapi: "3.0.0" +info: + version: 1.0.0 + title: Swagger Petstore + license: + name: MIT +servers: + - url: http://petstore.swagger.io/v1 +paths: + /pets: + get: + $ref: "./paths/pets/list.yaml" +`; + +const list = ` +--- +summary: List all pets +operationId: listPets +tags: + - pets +responses: + '200': + description: A paged array of pets + content: + application/json: + schema: + type: string +`; + +const expectedResult = ` +openapi: 3.0.0 +info: + version: 1.0.0 + title: Swagger Petstore + license: + name: MIT +servers: + - url: http://petstore.swagger.io/v1 +paths: + /pets: + get: + summary: List all pets + operationId: listPets + tags: + - pets + responses: + "200": + description: A paged array of pets + content: + application/json: + schema: + type: string +`; + +describe('bundleOpenApiSpecification', () => { + const readUrl = jest.fn(); + const reader = { + readUrl, + read: jest.fn(), + readTree: jest.fn(), + search: jest.fn(), + }; + + const scmIntegration = ScmIntegrations.fromConfig(new ConfigReader({})).byUrl( + 'https://github.com/owner/repo/blob/main/openapi.yaml', + ); + + beforeEach(() => { + jest.clearAllMocks(); + }); + + it('should return undefined if no specification is supplied', async () => { + expect( + await bundleOpenApiSpecification( + undefined, + 'https://github.com/owner/repo/blob/main/openapi.yaml', + reader, + scmIntegration as any, + ), + ).toBeUndefined(); + }); + + it('should return the bundled specification', async () => { + readUrl.mockResolvedValue({ + buffer: jest.fn().mockResolvedValue(list), + }); + + const result = await bundleOpenApiSpecification( + specification, + 'https://github.com/owner/repo/blob/main/openapi.yaml', + reader, + scmIntegration as any, + ); + + expect(result).toEqual(expectedResult.trimStart()); + }); +}); diff --git a/plugins/catalog-backend-module-openapi/src/lib/bundle.ts b/plugins/catalog-backend-module-openapi/src/lib/bundle.ts new file mode 100644 index 0000000000..3d35429d8b --- /dev/null +++ b/plugins/catalog-backend-module-openapi/src/lib/bundle.ts @@ -0,0 +1,69 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { UrlReader } from '@backstage/backend-common'; +import { ScmIntegration } from '@backstage/integration'; +import SwaggerParser from '@apidevtools/swagger-parser'; +import { parse, stringify } from 'yaml'; +import * as path from 'path'; + +const protocolPattern = /^(\w{2,}):\/\//i; +const getProtocol = (refPath: string) => { + const match = protocolPattern.exec(refPath); + if (match) { + return match[1].toLowerCase(); + } + return undefined; +}; + +export async function bundleOpenApiSpecification( + specification: string | undefined, + targetUrl: string, + reader: UrlReader, + scmIntegration: ScmIntegration, +): Promise { + const fileUrlReaderResolver: SwaggerParser.ResolverOptions = { + canRead: file => { + const protocol = getProtocol(file.url); + return protocol === undefined || protocol === 'file'; + }, + read: async file => { + const relativePath = path.relative('.', file.url); + const url = scmIntegration.resolveUrl({ + base: targetUrl, + url: relativePath, + }); + if (reader.readUrl) { + const data = await reader.readUrl(url); + return data.buffer(); + } + throw new Error('UrlReader has no readUrl method defined'); + }, + }; + + if (!specification) { + return undefined; + } + + const options: SwaggerParser.Options = { + resolve: { + file: fileUrlReaderResolver, + http: true, + }, + }; + const specObject = parse(specification); + const bundledSpec = await SwaggerParser.bundle(specObject, options); + return stringify(bundledSpec); +} diff --git a/plugins/catalog-backend-module-openapi/src/lib/index.ts b/plugins/catalog-backend-module-openapi/src/lib/index.ts new file mode 100644 index 0000000000..805569181a --- /dev/null +++ b/plugins/catalog-backend-module-openapi/src/lib/index.ts @@ -0,0 +1,16 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +export * from './bundle'; diff --git a/plugins/catalog-backend-module-openapi/src/setupTests.ts b/plugins/catalog-backend-module-openapi/src/setupTests.ts new file mode 100644 index 0000000000..d3232290a7 --- /dev/null +++ b/plugins/catalog-backend-module-openapi/src/setupTests.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export {}; diff --git a/yarn.lock b/yarn.lock index cf4f33b1d9..4a6ac9daf7 100644 --- a/yarn.lock +++ b/yarn.lock @@ -16,7 +16,7 @@ dependencies: "@jridgewell/trace-mapping" "^0.3.0" -"@apidevtools/json-schema-ref-parser@^9.0.6": +"@apidevtools/json-schema-ref-parser@9.0.6", "@apidevtools/json-schema-ref-parser@^9.0.6": version "9.0.6" resolved "https://registry.npmjs.org/@apidevtools/json-schema-ref-parser/-/json-schema-ref-parser-9.0.6.tgz#5d9000a3ac1fd25404da886da6b266adcd99cf1c" integrity sha512-M3YgsLjI0lZxvrpeGVk9Ap032W6TPQkH6pRAZz81Ac3WUNF79VQooAFnp8umjvVzUmD93NkogxEwbSce7qMsUg== @@ -25,6 +25,29 @@ call-me-maybe "^1.0.1" js-yaml "^3.13.1" +"@apidevtools/openapi-schemas@^2.1.0": + version "2.1.0" + resolved "https://registry.npmjs.org/@apidevtools/openapi-schemas/-/openapi-schemas-2.1.0.tgz#9fa08017fb59d80538812f03fc7cac5992caaa17" + integrity sha512-Zc1AlqrJlX3SlpupFGpiLi2EbteyP7fXmUOGup6/DnkRgjP9bgMM/ag+n91rsv0U1Gpz0H3VILA/o3bW7Ua6BQ== + +"@apidevtools/swagger-methods@^3.0.2": + version "3.0.2" + resolved "https://registry.npmjs.org/@apidevtools/swagger-methods/-/swagger-methods-3.0.2.tgz#b789a362e055b0340d04712eafe7027ddc1ac267" + integrity sha512-QAkD5kK2b1WfjDS/UQn/qQkbwF31uqRjPTrsCs5ZG9BQGAkjwvqGFjjPqAuzac/IYzpPtRzjCP1WrTuAIjMrXg== + +"@apidevtools/swagger-parser@^10.1.0": + version "10.1.0" + resolved "https://registry.npmjs.org/@apidevtools/swagger-parser/-/swagger-parser-10.1.0.tgz#a987d71e5be61feb623203be0c96e5985b192ab6" + integrity sha512-9Kt7EuS/7WbMAUv2gSziqjvxwDbFSg3Xeyfuj5laUODX8o/k/CpsAKiQ8W7/R88eXFTMbJYg6+7uAmOWNKmwnw== + dependencies: + "@apidevtools/json-schema-ref-parser" "9.0.6" + "@apidevtools/openapi-schemas" "^2.1.0" + "@apidevtools/swagger-methods" "^3.0.2" + "@jsdevtools/ono" "^7.1.3" + ajv "^8.6.3" + ajv-draft-04 "^1.0.0" + call-me-maybe "^1.0.1" + "@apollo/protobufjs@1.2.2": version "1.2.2" resolved "https://registry.npmjs.org/@apollo/protobufjs/-/protobufjs-1.2.2.tgz#4bd92cd7701ccaef6d517cdb75af2755f049f87c" @@ -7613,6 +7636,11 @@ aggregate-error@^3.0.0, aggregate-error@^3.1.0: clean-stack "^2.0.0" indent-string "^4.0.0" +ajv-draft-04@^1.0.0: + version "1.0.0" + resolved "https://registry.npmjs.org/ajv-draft-04/-/ajv-draft-04-1.0.0.tgz#3b64761b268ba0b9e668f0b41ba53fce0ad77fc8" + integrity sha512-mv00Te6nmYbRp5DCwclxtt7yV/joXJPGS7nM+97GdxvuttCOfgI3K4U25zboyeX0O+myI8ERluxQe5wljMmVIw== + ajv-formats@^2.1.1: version "2.1.1" resolved "https://registry.npmjs.org/ajv-formats/-/ajv-formats-2.1.1.tgz#6e669400659eb74973bbf2e33327180a0996b520" @@ -7642,7 +7670,7 @@ ajv@^6.10.0, ajv@^6.10.1, ajv@^6.12.2, ajv@^6.12.4, ajv@^6.12.5, ajv@^6.5.5, ajv json-schema-traverse "^0.4.1" uri-js "^4.2.2" -ajv@^8.0.0, ajv@^8.10.0, ajv@^8.8.0: +ajv@^8.0.0, ajv@^8.10.0, ajv@^8.6.3, ajv@^8.8.0: version "8.11.0" resolved "https://registry.npmjs.org/ajv/-/ajv-8.11.0.tgz#977e91dd96ca669f54a11e23e378e33b884a565f" integrity sha512-wGgprdCvMalC0BztXvitD2hC04YffAvtsUn93JbGXYLAtCUO4xd17mCCZQxUOItiBwZvJScWo8NIvQMQ71rdpg== @@ -19627,6 +19655,11 @@ openapi-sampler@^1.2.1: "@types/json-schema" "^7.0.7" json-pointer "0.6.2" +openapi-types@^11.0.1: + version "11.1.0" + resolved "https://registry.npmjs.org/openapi-types/-/openapi-types-11.1.0.tgz#037969f3dfa5999423ee33bf889fb0d12984277e" + integrity sha512-ZW+Jf12flFF6DXSij8DGL3svDA4RtSyHXjC/xB/JAh18gg3uVfVIFLvCfScUMowrpvlkxsMMbErakbth2g3/iQ== + openid-client@^4.1.1: version "4.9.0" resolved "https://registry.npmjs.org/openid-client/-/openid-client-4.9.0.tgz#bdfc9194435316df419f759ce177635146b43074"