Merge pull request #12814 from mfrinnstrom/openapi-basepath

OpenApiRefProcessor: add basepath support for $ref

.changeset/strong-birds-pretend.md

@@ -0,0 +1,21 @@
+---
+'@backstage/plugin-catalog-backend-module-openapi': patch
+---
+
+Add an `$openapi` placeholder resolver that supports more use cases for resolving `$ref` instances. This means that the quite recently added `OpenApiRefProcessor` has been deprecated in favor of the `openApiPlaceholderResolver`.
+
+An example of how to use it can be seen below.
+
+```yaml
+apiVersion: backstage.io/v1alpha1
+kind: API
+metadata:
+  name: example
+  description: Example API
+spec:
+  type: openapi
+  lifecycle: production
+  owner: team
+  definition:
+    $openapi: ./spec/openapi.yaml # by using $openapi Backstage will now resolve all $ref instances
+```

plugins/catalog-backend-module-openapi/README.md

@@ -15,6 +15,32 @@ yarn add --cwd packages/backend @backstage/plugin-catalog-backend-module-openapi
 
 ### Adding the plugin to your `packages/backend`
 
+#### **openApiPlaceholderResolver**
+
+The placeholder resolver can be added by importing `openApiPlaceholderResolver` in `src/plugins/catalog.ts` in your `backend` package and adding the following.
+
+```ts
+builder.setPlaceholderResolver('openapi', openApiPlaceholderResolver);
+```
+
+This allows you to use the `$openapi` placeholder when referencing your OpenAPI specification. This will then resolve all `$ref` instances in your specification.
+
+```yaml
+apiVersion: backstage.io/v1alpha1
+kind: API
+metadata:
+  name: example
+  description: Example API
+spec:
+  type: openapi
+  lifecycle: production
+  owner: team
+  definition:
+    $openapi: ./spec/openapi.yaml # by using $openapi Backstage will now resolve all $ref instances
+```
+
+#### **OpenAPIRefProcessor** (deprecated)
+
 The processor can be added by importing `OpenApiRefProcessor` in `src/plugins/catalog.ts` in your `backend` package and adding the following.
 
 ```ts

plugins/catalog-backend-module-openapi/api-report.md

@@ -6,12 +6,19 @@
 import { CatalogProcessor } from '@backstage/plugin-catalog-backend';
 import { Config } from '@backstage/config';
 import { Entity } from '@backstage/catalog-model';
+import { JsonValue } from '@backstage/types';
 import { LocationSpec } from '@backstage/plugin-catalog-backend';
 import { Logger } from 'winston';
+import { PlaceholderResolverParams } from '@backstage/plugin-catalog-backend';
 import { ScmIntegrations } from '@backstage/integration';
 import { UrlReader } from '@backstage/backend-common';
 
 // @public (undocumented)
+export function openApiPlaceholderResolver(
+  params: PlaceholderResolverParams,
+): Promise<JsonValue>;
+
+// @public @deprecated (undocumented)
 export class OpenApiRefProcessor implements CatalogProcessor {
   constructor(options: {
     integrations: ScmIntegrations;

plugins/catalog-backend-module-openapi/package.json

@@ -39,6 +39,8 @@
     "@backstage/config": "^1.0.1",
     "@backstage/integration": "^1.3.0-next.0",
     "@backstage/plugin-catalog-backend": "^1.3.1-next.0",
+    "@backstage/plugin-catalog-node": "^1.0.1-next.0",
+    "@backstage/types": "^1.0.0",
     "winston": "^3.2.1",
     "yaml": "^2.1.1"
   },

plugins/catalog-backend-module-openapi/src/OpenApiRefProcessor.test.ts

@@ -86,7 +86,7 @@ describe('OpenApiRefProcessor', () => {
 
     it('should ignore other specification types', async () => {
       const { entity, processor } = setupTest({
-        kind: 'Group',
+        kind: 'API',
         spec: { type: 'asyncapi' },
       });
 

plugins/catalog-backend-module-openapi/src/OpenApiRefProcessor.ts

@@ -24,7 +24,10 @@ import {
 import { bundleOpenApiSpecification } from './lib';
 import { Logger } from 'winston';
 
-/** @public */
+/**
+ * @public
+ * @deprecated replaced by the openApiPlaceholderResolver
+ */
 export class OpenApiRefProcessor implements CatalogProcessor {
   private readonly integrations: ScmIntegrations;
   private readonly logger: Logger;
@@ -69,17 +72,23 @@ export class OpenApiRefProcessor implements CatalogProcessor {
     }
 
     const scmIntegration = this.integrations.byUrl(location.target);
-    if (!scmIntegration) {
+    const definition = entity.spec!.definition;
+
+    if (!scmIntegration || !definition) {
       return entity;
     }
 
+    const resolveUrl = (url: string, base: string): string => {
+      return scmIntegration.resolveUrl({ url, base });
+    };
+
     this.logger.debug(`Bundling OpenAPI specification from ${location.target}`);
     try {
       const bundledSpec = await bundleOpenApiSpecification(
-        entity.spec!.definition?.toString(),
+        definition.toString(),
         location.target,
-        this.reader,
-        scmIntegration,
+        this.reader.read,
+        resolveUrl,
       );
 
       return {

plugins/catalog-backend-module-openapi/src/index.ts

@@ -14,3 +14,4 @@
  * limitations under the License.
  */
 export { OpenApiRefProcessor } from './OpenApiRefProcessor';
+export { openApiPlaceholderResolver } from './openApiPlaceholderResolver';

plugins/catalog-backend-module-openapi/src/lib/bundle.test.ts

@@ -13,8 +13,6 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import { ConfigReader } from '@backstage/config';
-import { ScmIntegrations } from '@backstage/integration';
 import { bundleOpenApiSpecification } from './bundle';
 
 const specification = `
@@ -73,43 +71,21 @@ paths:
 `;
 
 describe('bundleOpenApiSpecification', () => {
-  const readUrl = jest.fn();
-  const reader = {
-    readUrl,
-    read: jest.fn(),
-    readTree: jest.fn(),
-    search: jest.fn(),
-  };
-
-  const scmIntegration = ScmIntegrations.fromConfig(new ConfigReader({})).byUrl(
-    'https://github.com/owner/repo/blob/main/openapi.yaml',
-  );
+  const read = jest.fn();
+  const resolveUrl = jest.fn();
 
   beforeEach(() => {
     jest.clearAllMocks();
   });
 
-  it('should return undefined if no specification is supplied', async () => {
-    expect(
-      await bundleOpenApiSpecification(
-        undefined,
-        'https://github.com/owner/repo/blob/main/openapi.yaml',
-        reader,
-        scmIntegration as any,
-      ),
-    ).toBeUndefined();
-  });
-
   it('should return the bundled specification', async () => {
-    readUrl.mockResolvedValue({
-      buffer: jest.fn().mockResolvedValue(list),
-    });
+    read.mockResolvedValue(list);
 
     const result = await bundleOpenApiSpecification(
       specification,
-      'https://github.com/owner/repo/blob/main/openapi.yaml',
-      reader,
-      scmIntegration as any,
+      'https://github.com/owner/repo/blob/main/catalog-info.yaml',
+      read,
+      resolveUrl,
     );
 
     expect(result).toEqual(expectedResult.trimStart());

plugins/catalog-backend-module-openapi/src/lib/bundle.ts

@@ -13,8 +13,6 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import { UrlReader } from '@backstage/backend-common';
-import { ScmIntegration } from '@backstage/integration';
 import SwaggerParser from '@apidevtools/swagger-parser';
 import { parse, stringify } from 'yaml';
 import * as path from 'path';
@@ -28,12 +26,16 @@ const getProtocol = (refPath: string) => {
   return undefined;
 };
 
+export type BundlerRead = (url: string) => Promise<Buffer>;
+
+export type BundlerResolveUrl = (url: string, base: string) => string;
+
 export async function bundleOpenApiSpecification(
-  specification: string | undefined,
-  targetUrl: string,
-  reader: UrlReader,
-  scmIntegration: ScmIntegration,
-): Promise<string | undefined> {
+  specification: string,
+  baseUrl: string,
+  read: BundlerRead,
+  resolveUrl: BundlerResolveUrl,
+): Promise<string> {
   const fileUrlReaderResolver: SwaggerParser.ResolverOptions = {
     canRead: file => {
       const protocol = getProtocol(file.url);
@@ -41,22 +43,11 @@ export async function bundleOpenApiSpecification(
     },
     read: async file => {
       const relativePath = path.relative('.', file.url);
-      const url = scmIntegration.resolveUrl({
-        base: targetUrl,
-        url: relativePath,
-      });
-      if (reader.readUrl) {
-        const data = await reader.readUrl(url);
-        return data.buffer();
-      }
-      throw new Error('UrlReader has no readUrl method defined');
+      const url = resolveUrl(relativePath, baseUrl);
+      return await read(url);
     },
   };
 
-  if (!specification) {
-    return undefined;
-  }
-
   const options: SwaggerParser.Options = {
     resolve: {
       file: fileUrlReaderResolver,

plugins/catalog-backend-module-openapi/src/openApiPlaceholderResolver.test.ts

@@ -0,0 +1,63 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { PlaceholderResolverParams } from '@backstage/plugin-catalog-backend';
+import { openApiPlaceholderResolver } from './openApiPlaceholderResolver';
+import { bundleOpenApiSpecification } from './lib';
+
+jest.mock('./lib', () => ({
+  bundleOpenApiSpecification: jest.fn(),
+}));
+
+const bundledSpecification = '<bundled-specification>';
+
+describe('openApiPlaceholderResolver', () => {
+  const mockResolveUrl = jest.fn();
+  mockResolveUrl.mockReturnValue('mockUrl');
+
+  const mockRead = jest.fn();
+  mockRead.mockResolvedValue(Buffer.from('mockData'));
+
+  const params: PlaceholderResolverParams = {
+    key: 'openapi',
+    value: './spec/openapi.yaml',
+    baseUrl: 'https://github.com/owner/repo/blob/main/catalog-info.yaml',
+    resolveUrl: mockResolveUrl,
+    read: mockRead,
+    emit: jest.fn(),
+  };
+
+  beforeEach(() => {
+    (bundleOpenApiSpecification as any).mockResolvedValue(bundledSpecification);
+  });
+
+  afterEach(() => {
+    jest.clearAllMocks();
+  });
+
+  it('should throw error if unable to bundle the OpenAPI specification', async () => {
+    (bundleOpenApiSpecification as any).mockRejectedValue(new Error('TEST'));
+
+    await expect(openApiPlaceholderResolver(params)).rejects.toThrow(
+      'Placeholder $openapi unable to bundle OpenAPI specification',
+    );
+  });
+
+  it('should bundle the OpenAPI specification', async () => {
+    const result = await openApiPlaceholderResolver(params);
+
+    expect(result).toEqual(bundledSpecification);
+  });
+});

plugins/catalog-backend-module-openapi/src/openApiPlaceholderResolver.ts

@@ -0,0 +1,85 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { PlaceholderResolverParams } from '@backstage/plugin-catalog-backend';
+import { JsonValue } from '@backstage/types';
+import { processingResult } from '@backstage/plugin-catalog-node';
+import { bundleOpenApiSpecification } from './lib';
+
+/** @public */
+export async function openApiPlaceholderResolver(
+  params: PlaceholderResolverParams,
+): Promise<JsonValue> {
+  const { content, url } = await readTextLocation(params);
+
+  params.emit(processingResult.refresh(`url:${url}`));
+
+  try {
+    return await bundleOpenApiSpecification(
+      content,
+      url,
+      params.read,
+      params.resolveUrl,
+    );
+  } catch (error) {
+    throw new Error(
+      `Placeholder \$${params.key} unable to bundle OpenAPI specification at ${params.value}, ${error}`,
+    );
+  }
+}
+
+/*
+ * Helpers, copied from PlaceholderProcessor
+ */
+
+async function readTextLocation(
+  params: PlaceholderResolverParams,
+): Promise<{ content: string; url: string }> {
+  const newUrl = relativeUrl(params);
+
+  try {
+    const data = await params.read(newUrl);
+    return { content: data.toString('utf-8'), url: newUrl };
+  } catch (e) {
+    throw new Error(
+      `Placeholder \$${params.key} could not read location ${params.value}, ${e}`,
+    );
+  }
+}
+
+function relativeUrl({
+  key,
+  value,
+  baseUrl,
+  resolveUrl,
+}: PlaceholderResolverParams): string {
+  if (typeof value !== 'string') {
+    throw new Error(
+      `Placeholder \$${key} expected a string value parameter, in the form of an absolute URL or a relative path`,
+    );
+  }
+
+  try {
+    return resolveUrl(value, baseUrl);
+  } catch (e) {
+    // The only remaining case that isn't support is a relative file path that should be
+    // resolved using a relative file location. Accessing local file paths can lead to
+    // path traversal attacks and access to any file on the host system. Implementing this
+    // would require additional security measures.
+    throw new Error(
+      `Placeholder \$${key} could not form a URL out of ${baseUrl} and ${value}, ${e}`,
+    );
+  }
+}