feat: add ApiDefinitionAtLocationProcessor that allows to load a API definition from another location

Resolves #2572

docs/features/software-catalog/well-known-annotations.md

@@ -41,6 +41,27 @@ expecting a two-item array out of it. The format of the target part is
 type-dependent and could conceivably even be an empty string, but the separator
 colon is always present.
 
+### backstage.io/definition-at-location
+
+```yaml
+# Example
+apiVersion: backstage.io/v1alpha1
+kind: API
+metadata:
+  name: petstore
+  annotations:
+    backstage.io/definition-at-location: 'url:https://petstore.swagger.io/v2/swagger.json'
+spec:
+  type: openapi
+```
+
+This annotation allows to fetch an API definition from another location, instead
+of wrapping the API definition inside the definition field. This allows to
+easitly consume existing API definition. The definition is fetched during
+ingestion by a processor and included in the entity. It is updated on every
+refresh. The annotation contains a location reference string that contains the
+location processor type and the target.
+
 ### backstage.io/techdocs-ref
 
 ```yaml

packages/catalog-model/examples/all-apis.yaml

@@ -8,3 +8,5 @@ spec:
   targets:
     - https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/hello-world-api.yaml
     - https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/streetlights-api.yaml
+    - https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/spotify-api.yaml
+    - https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/swapi-graphql.yaml

packages/catalog-model/examples/spotify-api.yaml

@@ -0,0 +1,14 @@
+apiVersion: backstage.io/v1alpha1
+kind: API
+metadata:
+  name: spotify
+  description: The Spotify web API
+  tags:
+    - spotify
+    - rest
+  annotations:
+    backstage.io/definition-at-location: 'url:https://raw.githubusercontent.com/APIs-guru/openapi-directory/master/APIs/spotify.com/v1/swagger.yaml'
+spec:
+  type: openapi
+  lifecycle: production
+  owner: spotify@example.com

plugins/catalog-backend/src/ingestion/LocationReaders.ts

@@ -47,6 +47,7 @@ import {
 import { YamlProcessor } from './processors/YamlProcessor';
 import { LocationReader, ReadLocationResult } from './types';
 import { CatalogRulesEnforcer } from './CatalogRules';
+import { ApiDefinitionAtLocationProcessor } from './processors/ApiDefinitionAtLocationProcessor';
 
 // The max amount of nesting depth of generated work items
 const MAX_DEPTH = 10;
@@ -85,6 +86,7 @@ export class LocationReaders implements LocationReader {
       new AzureApiReaderProcessor(config),
       new UrlReaderProcessor(),
       new YamlProcessor(),
+      new ApiDefinitionAtLocationProcessor(),
       new EntityPolicyProcessor(entityPolicy),
       new LocationRefProcessor(),
       new AnnotateLocationEntityProcessor(),
@@ -218,7 +220,12 @@ export class LocationReaders implements LocationReader {
     for (const processor of this.processors) {
       if (processor.processEntity) {
         try {
-          current = await processor.processEntity(current, item.location, emit);
+          current = await processor.processEntity(
+            current,
+            item.location,
+            emit,
+            this.readLocation.bind(this),
+          );
         } catch (e) {
           const message = `Processor ${processor.constructor.name} threw an error while processing entity at ${item.location.type} ${item.location.target}, ${e}`;
           emit(result.generalError(item.location, message));
@@ -248,4 +255,25 @@ export class LocationReaders implements LocationReader {
       }
     }
   }
+
+  private async readLocation(
+    location: LocationSpec,
+  ): Promise<LocationProcessorResult> {
+    let locationResult: LocationProcessorResult | undefined;
+
+    await this.handleLocation(
+      {
+        type: 'location',
+        location,
+        optional: false,
+      },
+      r => (locationResult = r),
+    );
+
+    if (!locationResult) {
+      throw new Error('No location loaded');
+    }
+
+    return locationResult;
+  }
 }

plugins/catalog-backend/src/ingestion/processors/ApiDefinitionAtLocationProcessor.test.ts

@@ -0,0 +1,132 @@
+import { ApiEntity, Entity, LocationSpec } from '@backstage/catalog-model';
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { ApiDefinitionAtLocationProcessor } from './ApiDefinitionAtLocationProcessor';
+import { LocationProcessorResult } from './types';
+
+describe('ApiDefinitionAtLocationProcessor', () => {
+  let processor: ApiDefinitionAtLocationProcessor;
+  let entity: Entity;
+  let location: LocationSpec;
+
+  beforeEach(() => {
+    processor = new ApiDefinitionAtLocationProcessor();
+    entity = {
+      apiVersion: 'backstage.io/v1alpha1',
+      kind: 'API',
+      metadata: {
+        name: 'test',
+      },
+      spec: {
+        lifecycle: 'production',
+        owner: 'info@example.com',
+        type: 'openapi',
+        definition: 'Hello',
+      },
+    };
+    location = {
+      type: 'url',
+      target: `http://example.com/api.yaml`,
+    };
+  });
+
+  it('should skip entities without annotation', async () => {
+    const read = jest.fn(
+      (): Promise<LocationProcessorResult> => {
+        throw new Error();
+      },
+    );
+
+    const generated = (await processor.processEntity(
+      entity,
+      location,
+      () => {},
+      read,
+    )) as ApiEntity;
+
+    expect(generated.spec.definition).toBe('Hello');
+  });
+
+  it('should load from location', async () => {
+    entity.metadata.annotations = {
+      'backstage.io/definition-at-location':
+        'url:http://example.com/openapi.yaml',
+    };
+
+    const read = jest.fn(
+      (l: LocationSpec): Promise<LocationProcessorResult> =>
+        Promise.resolve({
+          type: 'data',
+          data: Buffer.from('Hello'),
+          location: l,
+        }),
+    );
+
+    const generated = (await processor.processEntity(
+      entity,
+      location,
+      () => {},
+      read,
+    )) as ApiEntity;
+
+    expect(generated.spec.definition).toBe('Hello');
+    expect(read.mock.calls[0][0]).toStrictEqual({
+      type: 'url',
+      target: 'http://example.com/openapi.yaml',
+    });
+  });
+
+  it('should throw errors while loading', async () => {
+    entity.metadata.annotations = {
+      'backstage.io/definition-at-location': 'missing',
+    };
+
+    const read = jest.fn(
+      (l: LocationSpec): Promise<LocationProcessorResult> =>
+        Promise.resolve({
+          type: 'error',
+          error: new Error('Failed to load location'),
+          location: l,
+        }),
+    );
+
+    await expect(
+      processor.processEntity(entity, location, () => {}, read),
+    ).rejects.toThrow('Failed to read location: Failed to load location');
+  });
+
+  it('should throw errors if location read has wrong type', async () => {
+    entity.metadata.annotations = {
+      'backstage.io/definition-at-location': 'wrong',
+    };
+
+    const read = jest.fn(
+      (l: LocationSpec): Promise<LocationProcessorResult> =>
+        Promise.resolve({
+          type: 'location',
+          optional: false,
+          location: l,
+        }),
+    );
+
+    await expect(
+      processor.processEntity(entity, location, () => {}, read),
+    ).rejects.toThrow(
+      `Only supports location processor results of type 'data', but got 'location'`,
+    );
+  });
+});

plugins/catalog-backend/src/ingestion/processors/ApiDefinitionAtLocationProcessor.ts

@@ -0,0 +1,70 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { ApiEntity, Entity, LocationSpec } from '@backstage/catalog-model';
+import {
+  LocationProcessor,
+  LocationProcessorEmit,
+  LocationProcessorRead,
+} from './types';
+
+const DEFINITION_AT_LOCATION_ANNOTATION = 'backstage.io/definition-at-location';
+
+export class ApiDefinitionAtLocationProcessor implements LocationProcessor {
+  async processEntity(
+    entity: Entity,
+    _location: LocationSpec,
+    _emit: LocationProcessorEmit,
+    read: LocationProcessorRead,
+  ): Promise<Entity> {
+    if (
+      entity.kind !== 'API' ||
+      !entity.metadata.annotations ||
+      !entity.metadata.annotations[DEFINITION_AT_LOCATION_ANNOTATION]
+    ) {
+      return entity;
+    }
+
+    const reference =
+      entity.metadata.annotations[DEFINITION_AT_LOCATION_ANNOTATION];
+    const { type, target } = extractReference(reference);
+    const result = await read({ type, target });
+
+    if (result.type === 'error') {
+      throw new Error(`Failed to read location: ${result.error.message}`);
+    }
+
+    if (result.type !== 'data') {
+      throw new Error(
+        `Only supports location processor results of type 'data', but got '${result.type}'`,
+      );
+    }
+
+    const definition = result.data.toString();
+    const apiEntity = entity as ApiEntity;
+    apiEntity.spec.definition = definition;
+
+    return entity;
+  }
+}
+
+function extractReference(reference: string): { type: string; target: string } {
+  const delimiterIndex = reference.indexOf(':');
+  const type = reference.slice(0, delimiterIndex);
+  const target = reference.slice(delimiterIndex + 1);
+
+  return { type, target };
+}

plugins/catalog-backend/src/ingestion/processors/types.ts

@@ -50,6 +50,7 @@ export type LocationProcessor = {
    *
    * @param entity The entity to process
    * @param location The location that the entity came from
+   * @param read Reads the contents of a location
    * @param emit A sink for auxiliary items resulting from the processing
    * @returns The same entity or a modifid version of it
    */
@@ -57,6 +58,7 @@ export type LocationProcessor = {
     entity: Entity,
     location: LocationSpec,
     emit: LocationProcessorEmit,
+    read: LocationProcessorRead,
   ): Promise<Entity>;
 
   /**
@@ -107,3 +109,7 @@ export type LocationProcessorResult =
   | LocationProcessorDataResult
   | LocationProcessorEntityResult
   | LocationProcessorErrorResult;
+
+export type LocationProcessorRead = (
+  location: LocationSpec,
+) => Promise<LocationProcessorResult>;