From b50e8e533b7aca0c93b78a56b2788cf7209a4b4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mattias=20Frinnstr=C3=B6m?= Date: Thu, 11 Aug 2022 12:04:27 +0200 Subject: [PATCH] Add openApiPlaceholderResolver MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Mattias Frinnström --- .changeset/strong-birds-pretend.md | 21 +++++ .../catalog-backend-module-openapi/README.md | 26 ++++++ .../api-report.md | 7 ++ .../package.json | 2 + .../src/OpenApiRefProcessor.test.ts | 2 +- .../src/OpenApiRefProcessor.ts | 19 +++-- .../src/index.ts | 1 + .../src/lib/bundle.test.ts | 36 ++------ .../src/lib/bundle.ts | 31 +++---- .../src/openApiPlaceholderResolver.test.ts | 63 ++++++++++++++ .../src/openApiPlaceholderResolver.ts | 85 +++++++++++++++++++ 11 files changed, 237 insertions(+), 56 deletions(-) create mode 100644 .changeset/strong-birds-pretend.md create mode 100644 plugins/catalog-backend-module-openapi/src/openApiPlaceholderResolver.test.ts create mode 100644 plugins/catalog-backend-module-openapi/src/openApiPlaceholderResolver.ts diff --git a/.changeset/strong-birds-pretend.md b/.changeset/strong-birds-pretend.md new file mode 100644 index 0000000000..9ad3bae97b --- /dev/null +++ b/.changeset/strong-birds-pretend.md @@ -0,0 +1,21 @@ +--- +'@backstage/plugin-catalog-backend-module-openapi': patch +--- + +Add an `$openapi` placeholder resolver that supports more use cases for resolving `$ref` instances. This means that the quite recently added `OpenApiRefProcessor` has been deprecated in favor of the `openApiPlaceholderResolver`. + +An example of how to use it can be seen below. + +```yaml +apiVersion: backstage.io/v1alpha1 +kind: API +metadata: + name: example + description: Example API +spec: + type: openapi + lifecycle: production + owner: team + definition: + $openapi: ./spec/openapi.yaml # by using $openapi Backstage will now resolve all $ref instances +``` diff --git a/plugins/catalog-backend-module-openapi/README.md b/plugins/catalog-backend-module-openapi/README.md index 1a133521c6..f80798e267 100644 --- a/plugins/catalog-backend-module-openapi/README.md +++ b/plugins/catalog-backend-module-openapi/README.md @@ -15,6 +15,32 @@ yarn add --cwd packages/backend @backstage/plugin-catalog-backend-module-openapi ### Adding the plugin to your `packages/backend` +#### **openApiPlaceholderResolver** + +The placeholder resolver can be added by importing `openApiPlaceholderResolver` in `src/plugins/catalog.ts` in your `backend` package and adding the following. + +```ts +builder.setPlaceholderResolver('openapi', openApiPlaceholderResolver); +``` + +This allows you to use the `$openapi` placeholder when referencing your OpenAPI specification. This will then resolve all `$ref` instances in your specification. + +```yaml +apiVersion: backstage.io/v1alpha1 +kind: API +metadata: + name: example + description: Example API +spec: + type: openapi + lifecycle: production + owner: team + definition: + $openapi: ./spec/openapi.yaml # by using $openapi Backstage will now resolve all $ref instances +``` + +#### **OpenAPIRefProcessor** (deprecated) + The processor can be added by importing `OpenApiRefProcessor` in `src/plugins/catalog.ts` in your `backend` package and adding the following. ```ts diff --git a/plugins/catalog-backend-module-openapi/api-report.md b/plugins/catalog-backend-module-openapi/api-report.md index 92ddd1e3cd..d3a40067c5 100644 --- a/plugins/catalog-backend-module-openapi/api-report.md +++ b/plugins/catalog-backend-module-openapi/api-report.md @@ -6,12 +6,19 @@ import { CatalogProcessor } from '@backstage/plugin-catalog-backend'; import { Config } from '@backstage/config'; import { Entity } from '@backstage/catalog-model'; +import { JsonValue } from '@backstage/types'; import { LocationSpec } from '@backstage/plugin-catalog-backend'; import { Logger } from 'winston'; +import { PlaceholderResolverParams } from '@backstage/plugin-catalog-backend'; import { ScmIntegrations } from '@backstage/integration'; import { UrlReader } from '@backstage/backend-common'; // @public (undocumented) +export function openApiPlaceholderResolver( + params: PlaceholderResolverParams, +): Promise; + +// @public @deprecated (undocumented) export class OpenApiRefProcessor implements CatalogProcessor { constructor(options: { integrations: ScmIntegrations; diff --git a/plugins/catalog-backend-module-openapi/package.json b/plugins/catalog-backend-module-openapi/package.json index d2d4d4989c..81f8875247 100644 --- a/plugins/catalog-backend-module-openapi/package.json +++ b/plugins/catalog-backend-module-openapi/package.json @@ -39,6 +39,8 @@ "@backstage/config": "^1.0.1", "@backstage/integration": "^1.3.0-next.0", "@backstage/plugin-catalog-backend": "^1.3.1-next.0", + "@backstage/plugin-catalog-node": "^1.0.1-next.0", + "@backstage/types": "^1.0.0", "winston": "^3.2.1", "yaml": "^2.1.1" }, diff --git a/plugins/catalog-backend-module-openapi/src/OpenApiRefProcessor.test.ts b/plugins/catalog-backend-module-openapi/src/OpenApiRefProcessor.test.ts index 8f2cc571c2..21080202b5 100644 --- a/plugins/catalog-backend-module-openapi/src/OpenApiRefProcessor.test.ts +++ b/plugins/catalog-backend-module-openapi/src/OpenApiRefProcessor.test.ts @@ -86,7 +86,7 @@ describe('OpenApiRefProcessor', () => { it('should ignore other specification types', async () => { const { entity, processor } = setupTest({ - kind: 'Group', + kind: 'API', spec: { type: 'asyncapi' }, }); diff --git a/plugins/catalog-backend-module-openapi/src/OpenApiRefProcessor.ts b/plugins/catalog-backend-module-openapi/src/OpenApiRefProcessor.ts index 01875e0779..f4cffdf48f 100644 --- a/plugins/catalog-backend-module-openapi/src/OpenApiRefProcessor.ts +++ b/plugins/catalog-backend-module-openapi/src/OpenApiRefProcessor.ts @@ -24,7 +24,10 @@ import { import { bundleOpenApiSpecification } from './lib'; import { Logger } from 'winston'; -/** @public */ +/** + * @public + * @deprecated replaced by the openApiPlaceholderResolver + */ export class OpenApiRefProcessor implements CatalogProcessor { private readonly integrations: ScmIntegrations; private readonly logger: Logger; @@ -69,17 +72,23 @@ export class OpenApiRefProcessor implements CatalogProcessor { } const scmIntegration = this.integrations.byUrl(location.target); - if (!scmIntegration) { + const definition = entity.spec!.definition; + + if (!scmIntegration || !definition) { return entity; } + const resolveUrl = (url: string, base: string): string => { + return scmIntegration.resolveUrl({ url, base }); + }; + this.logger.debug(`Bundling OpenAPI specification from ${location.target}`); try { const bundledSpec = await bundleOpenApiSpecification( - entity.spec!.definition?.toString(), + definition.toString(), location.target, - this.reader, - scmIntegration, + this.reader.read, + resolveUrl, ); return { diff --git a/plugins/catalog-backend-module-openapi/src/index.ts b/plugins/catalog-backend-module-openapi/src/index.ts index f98bee6bac..ccd3ced71e 100644 --- a/plugins/catalog-backend-module-openapi/src/index.ts +++ b/plugins/catalog-backend-module-openapi/src/index.ts @@ -14,3 +14,4 @@ * limitations under the License. */ export { OpenApiRefProcessor } from './OpenApiRefProcessor'; +export { openApiPlaceholderResolver } from './openApiPlaceholderResolver'; diff --git a/plugins/catalog-backend-module-openapi/src/lib/bundle.test.ts b/plugins/catalog-backend-module-openapi/src/lib/bundle.test.ts index 4505f158b5..0ecd9f7fd1 100644 --- a/plugins/catalog-backend-module-openapi/src/lib/bundle.test.ts +++ b/plugins/catalog-backend-module-openapi/src/lib/bundle.test.ts @@ -13,8 +13,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { ConfigReader } from '@backstage/config'; -import { ScmIntegrations } from '@backstage/integration'; import { bundleOpenApiSpecification } from './bundle'; const specification = ` @@ -73,43 +71,21 @@ paths: `; describe('bundleOpenApiSpecification', () => { - const readUrl = jest.fn(); - const reader = { - readUrl, - read: jest.fn(), - readTree: jest.fn(), - search: jest.fn(), - }; - - const scmIntegration = ScmIntegrations.fromConfig(new ConfigReader({})).byUrl( - 'https://github.com/owner/repo/blob/main/openapi.yaml', - ); + const read = jest.fn(); + const resolveUrl = jest.fn(); beforeEach(() => { jest.clearAllMocks(); }); - it('should return undefined if no specification is supplied', async () => { - expect( - await bundleOpenApiSpecification( - undefined, - 'https://github.com/owner/repo/blob/main/openapi.yaml', - reader, - scmIntegration as any, - ), - ).toBeUndefined(); - }); - it('should return the bundled specification', async () => { - readUrl.mockResolvedValue({ - buffer: jest.fn().mockResolvedValue(list), - }); + read.mockResolvedValue(list); const result = await bundleOpenApiSpecification( specification, - 'https://github.com/owner/repo/blob/main/openapi.yaml', - reader, - scmIntegration as any, + 'https://github.com/owner/repo/blob/main/catalog-info.yaml', + read, + resolveUrl, ); expect(result).toEqual(expectedResult.trimStart()); diff --git a/plugins/catalog-backend-module-openapi/src/lib/bundle.ts b/plugins/catalog-backend-module-openapi/src/lib/bundle.ts index 3d35429d8b..6ac8d211e2 100644 --- a/plugins/catalog-backend-module-openapi/src/lib/bundle.ts +++ b/plugins/catalog-backend-module-openapi/src/lib/bundle.ts @@ -13,8 +13,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { UrlReader } from '@backstage/backend-common'; -import { ScmIntegration } from '@backstage/integration'; import SwaggerParser from '@apidevtools/swagger-parser'; import { parse, stringify } from 'yaml'; import * as path from 'path'; @@ -28,12 +26,16 @@ const getProtocol = (refPath: string) => { return undefined; }; +export type BundlerRead = (url: string) => Promise; + +export type BundlerResolveUrl = (url: string, base: string) => string; + export async function bundleOpenApiSpecification( - specification: string | undefined, - targetUrl: string, - reader: UrlReader, - scmIntegration: ScmIntegration, -): Promise { + specification: string, + baseUrl: string, + read: BundlerRead, + resolveUrl: BundlerResolveUrl, +): Promise { const fileUrlReaderResolver: SwaggerParser.ResolverOptions = { canRead: file => { const protocol = getProtocol(file.url); @@ -41,22 +43,11 @@ export async function bundleOpenApiSpecification( }, read: async file => { const relativePath = path.relative('.', file.url); - const url = scmIntegration.resolveUrl({ - base: targetUrl, - url: relativePath, - }); - if (reader.readUrl) { - const data = await reader.readUrl(url); - return data.buffer(); - } - throw new Error('UrlReader has no readUrl method defined'); + const url = resolveUrl(relativePath, baseUrl); + return await read(url); }, }; - if (!specification) { - return undefined; - } - const options: SwaggerParser.Options = { resolve: { file: fileUrlReaderResolver, diff --git a/plugins/catalog-backend-module-openapi/src/openApiPlaceholderResolver.test.ts b/plugins/catalog-backend-module-openapi/src/openApiPlaceholderResolver.test.ts new file mode 100644 index 0000000000..f58a818ba5 --- /dev/null +++ b/plugins/catalog-backend-module-openapi/src/openApiPlaceholderResolver.test.ts @@ -0,0 +1,63 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { PlaceholderResolverParams } from '@backstage/plugin-catalog-backend'; +import { openApiPlaceholderResolver } from './openApiPlaceholderResolver'; +import { bundleOpenApiSpecification } from './lib'; + +jest.mock('./lib', () => ({ + bundleOpenApiSpecification: jest.fn(), +})); + +const bundledSpecification = ''; + +describe('openApiPlaceholderResolver', () => { + const mockResolveUrl = jest.fn(); + mockResolveUrl.mockReturnValue('mockUrl'); + + const mockRead = jest.fn(); + mockRead.mockResolvedValue(Buffer.from('mockData')); + + const params: PlaceholderResolverParams = { + key: 'openapi', + value: './spec/openapi.yaml', + baseUrl: 'https://github.com/owner/repo/blob/main/catalog-info.yaml', + resolveUrl: mockResolveUrl, + read: mockRead, + emit: jest.fn(), + }; + + beforeEach(() => { + (bundleOpenApiSpecification as any).mockResolvedValue(bundledSpecification); + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + it('should throw error if unable to bundle the OpenAPI specification', async () => { + (bundleOpenApiSpecification as any).mockRejectedValue(new Error('TEST')); + + await expect(openApiPlaceholderResolver(params)).rejects.toThrow( + 'Placeholder $openapi unable to bundle OpenAPI specification', + ); + }); + + it('should bundle the OpenAPI specification', async () => { + const result = await openApiPlaceholderResolver(params); + + expect(result).toEqual(bundledSpecification); + }); +}); diff --git a/plugins/catalog-backend-module-openapi/src/openApiPlaceholderResolver.ts b/plugins/catalog-backend-module-openapi/src/openApiPlaceholderResolver.ts new file mode 100644 index 0000000000..96fadb3cbc --- /dev/null +++ b/plugins/catalog-backend-module-openapi/src/openApiPlaceholderResolver.ts @@ -0,0 +1,85 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { PlaceholderResolverParams } from '@backstage/plugin-catalog-backend'; +import { JsonValue } from '@backstage/types'; +import { processingResult } from '@backstage/plugin-catalog-node'; +import { bundleOpenApiSpecification } from './lib'; + +/** @public */ +export async function openApiPlaceholderResolver( + params: PlaceholderResolverParams, +): Promise { + const { content, url } = await readTextLocation(params); + + params.emit(processingResult.refresh(`url:${url}`)); + + try { + return await bundleOpenApiSpecification( + content, + url, + params.read, + params.resolveUrl, + ); + } catch (error) { + throw new Error( + `Placeholder \$${params.key} unable to bundle OpenAPI specification at ${params.value}, ${error}`, + ); + } +} + +/* + * Helpers, copied from PlaceholderProcessor + */ + +async function readTextLocation( + params: PlaceholderResolverParams, +): Promise<{ content: string; url: string }> { + const newUrl = relativeUrl(params); + + try { + const data = await params.read(newUrl); + return { content: data.toString('utf-8'), url: newUrl }; + } catch (e) { + throw new Error( + `Placeholder \$${params.key} could not read location ${params.value}, ${e}`, + ); + } +} + +function relativeUrl({ + key, + value, + baseUrl, + resolveUrl, +}: PlaceholderResolverParams): string { + if (typeof value !== 'string') { + throw new Error( + `Placeholder \$${key} expected a string value parameter, in the form of an absolute URL or a relative path`, + ); + } + + try { + return resolveUrl(value, baseUrl); + } catch (e) { + // The only remaining case that isn't support is a relative file path that should be + // resolved using a relative file location. Accessing local file paths can lead to + // path traversal attacks and access to any file on the host system. Implementing this + // would require additional security measures. + throw new Error( + `Placeholder \$${key} could not form a URL out of ${baseUrl} and ${value}, ${e}`, + ); + } +}