Merge branch 'master' of github.com:spotify/backstage into ndudnik/use-catalog-backend

packages/backend/package.json

@@ -45,6 +45,9 @@
     "typescript": "^3.9.2"
   },
   "nodemonConfig": {
-    "watch": "./dist"
+    "watch": [
+      "./dist",
+      "node_modules/@backstage*"
+    ]
   }
 }

packages/catalog-model/src/setupTests.ts

@@ -0,0 +1,15 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */

plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.ts

@@ -14,9 +14,14 @@
  * limitations under the License.
  */
 
-import { Database } from '../database';
+import { Database, DatabaseLocationUpdateLogEvent } from '../database';
 import { IngestionModel } from '../ingestion/types';
-import { AddLocation, Location, LocationsCatalog } from './types';
+import {
+  AddLocation,
+  Location,
+  LocationsCatalog,
+  LocationResponse,
+} from './types';
 
 export class DatabaseLocationsCatalog implements LocationsCatalog {
   constructor(
@@ -50,13 +55,36 @@ export class DatabaseLocationsCatalog implements LocationsCatalog {
     await this.database.removeLocation(id);
   }
 
-  async locations(): Promise<Location[]> {
+  async locations(): Promise<LocationResponse[]> {
     const items = await this.database.locations();
-    return items;
+    return items.map(({ message, status, timestamp, ...data }) => ({
+      currentStatus: {
+        message,
+        status,
+        timestamp,
+      },
+      data,
+    }));
   }
 
-  async location(id: string): Promise<Location> {
-    const item = await this.database.location(id);
-    return item;
+  async locationHistory(id: string): Promise<DatabaseLocationUpdateLogEvent[]> {
+    return this.database.locationHistory(id);
+  }
+
+  async location(id: string): Promise<LocationResponse> {
+    const {
+      message,
+      status,
+      timestamp,
+      ...data
+    } = await this.database.location(id);
+    return {
+      currentStatus: {
+        message,
+        status,
+        timestamp,
+      },
+      data,
+    };
   }
 }

plugins/catalog-backend/src/catalog/types.ts

@@ -41,12 +41,31 @@ export type EntitiesCatalog = {
 // Locations
 //
 
+export type LocationUpdateStatus = {
+  timestamp: string | null;
+  status: string | null;
+  message: string | null;
+};
+export type LocationUpdateLogEvent = {
+  id: string;
+  status: 'fail' | 'success';
+  location_id: string;
+  entity_name: string;
+  created_at?: string;
+  message?: string;
+};
+
 export type Location = {
   id: string;
   type: string;
   target: string;
 };
 
+export type LocationResponse = {
+  data: Location;
+  currentStatus: LocationUpdateStatus;
+};
+
 export type AddLocation = {
   type: string;
   target: string;
@@ -62,6 +81,7 @@ export const addLocationSchema: yup.Schema<AddLocation> = yup
 export type LocationsCatalog = {
   addLocation(location: AddLocation): Promise<Location>;
   removeLocation(id: string): Promise<void>;
-  locations(): Promise<Location[]>;
-  location(id: string): Promise<Location>;
+  locations(): Promise<LocationResponse[]>;
+  location(id: string): Promise<LocationResponse>;
+  locationHistory(id: string): Promise<LocationUpdateLogEvent[]>;
 };

plugins/catalog-backend/src/database/Database.test.ts

@@ -27,6 +27,8 @@ import {
   Database,
   AddDatabaseLocation,
   DbLocationsRow,
+  DbLocationsRowWithStatus,
+  DatabaseLocationUpdateLogStatus,
 } from '.';
 import { Entity } from '@backstage/catalog-model';
 
@@ -74,7 +76,9 @@ describe('Database', () => {
           name: 'c',
           namespace: 'd',
           labels: { e: 'f' },
-          annotations: { g: 'h' },
+          annotations: {
+            g: 'h',
+          },
         },
         spec: { i: 'j' },
       },
@@ -84,10 +88,13 @@ describe('Database', () => {
   it('manages locations', async () => {
     const db = new Database(database, getVoidLogger());
     const input: AddDatabaseLocation = { type: 'a', target: 'b' };
-    const output: DbLocationsRow = {
+    const output: DbLocationsRowWithStatus = {
       id: expect.anything(),
       type: 'a',
       target: 'b',
+      message: null,
+      status: null,
+      timestamp: null,
     };
 
     await db.addLocation(input);
@@ -118,7 +125,7 @@ describe('Database', () => {
     // Output is the same
     expect(output2).toEqual(output1);
     // Locations contain only one record
-    expect(locations).toEqual([output1]);
+    expect(locations[0]).toMatchObject(output1);
   });
 
   describe('addEntity', () => {
@@ -150,6 +157,45 @@ describe('Database', () => {
     });
   });
 
+  describe('locationHistory', () => {
+    it('outputs the history correctly', async () => {
+      const catalog = new Database(database, getVoidLogger());
+      const location: AddDatabaseLocation = { type: 'a', target: 'b' };
+      const { id: locationId } = await catalog.addLocation(location);
+
+      await catalog.addLocationUpdateLogEvent(
+        locationId,
+        DatabaseLocationUpdateLogStatus.SUCCESS,
+      );
+      await catalog.addLocationUpdateLogEvent(
+        locationId,
+        DatabaseLocationUpdateLogStatus.FAIL,
+        undefined,
+        'Something went wrong',
+      );
+
+      const result = await catalog.locationHistory(locationId);
+      expect(result).toEqual([
+        {
+          created_at: expect.anything(),
+          entity_name: null,
+          id: expect.anything(),
+          location_id: locationId,
+          message: null,
+          status: DatabaseLocationUpdateLogStatus.SUCCESS,
+        },
+        {
+          created_at: expect.anything(),
+          entity_name: null,
+          id: expect.anything(),
+          location_id: locationId,
+          message: 'Something went wrong',
+          status: DatabaseLocationUpdateLogStatus.FAIL,
+        },
+      ]);
+    });
+  });
+
   describe('updateEntity', () => {
     it('can read and no-op-update an entity', async () => {
       const catalog = new Database(database, getVoidLogger());

plugins/catalog-backend/src/database/Database.ts

@@ -35,6 +35,7 @@ import {
   DbEntityRequest,
   DbEntityResponse,
   DbLocationsRow,
+  DbLocationsRowWithStatus,
 } from './types';
 
 function getStrippedMetadata(metadata: EntityMeta): EntityMeta {
@@ -182,6 +183,12 @@ export class Database {
       uid: generateUid(),
       etag: generateEtag(),
       generation: 1,
+      annotations: {
+        ...(newEntity.metadata?.annotations ?? {}),
+        ...(request.locationId
+          ? { 'backstage.io/managed-by-location': request.locationId }
+          : {}),
+      },
     };
 
     const newRow = toEntityRow(request.locationId, newEntity);
@@ -378,18 +385,52 @@ export class Database {
     }
   }
 
-  async location(id: string): Promise<DbLocationsRow> {
-    const items = await this.database<DbLocationsRow>('locations')
-      .where({ id })
-      .select();
+  async location(id: string): Promise<DbLocationsRowWithStatus> {
+    const items = await this.database<DbLocationsRowWithStatus>('locations')
+      .where('locations.id', id)
+      .leftOuterJoin(
+        'location_update_log_latest',
+        'locations.id',
+        'location_update_log_latest.location_id',
+      )
+      .select('locations.*', {
+        status: 'location_update_log_latest.status',
+        timestamp: 'location_update_log_latest.created_at',
+        message: 'location_update_log_latest.message',
+      });
+
     if (!items.length) {
       throw new NotFoundError(`Found no location with ID ${id}`);
     }
     return items[0];
   }
 
-  async locations(): Promise<DbLocationsRow[]> {
-    return this.database<DbLocationsRow>('locations').select();
+  async locations(): Promise<DbLocationsRowWithStatus[]> {
+    const locations = await this.database('locations')
+      .leftOuterJoin(
+        'location_update_log_latest',
+        'locations.id',
+        'location_update_log_latest.location_id',
+      )
+      .select('locations.*', {
+        status: 'location_update_log_latest.status',
+        timestamp: 'location_update_log_latest.created_at',
+        message: 'location_update_log_latest.message',
+      });
+
+    return locations;
+  }
+
+  async locationHistory(id: string): Promise<DatabaseLocationUpdateLogEvent[]> {
+    const result = await this.database<DatabaseLocationUpdateLogEvent>(
+      'location_update_log',
+    )
+      .where('location_id', id)
+      .orderBy('created_at', 'desc')
+      .limit(10)
+      .select();
+
+    return result;
   }
 
   async addLocationUpdateLogEvent(
@@ -402,7 +443,7 @@ export class Database {
       'location_update_log',
     ).insert({
       id: uuidv4(),
-      status: status,
+      status,
       location_id: locationId,
       entity_name: entityName,
       message,

plugins/catalog-backend/src/database/DatabaseManager.test.ts

@@ -18,7 +18,11 @@ import { getVoidLogger } from '@backstage/backend-common';
 import Knex from 'knex';
 import { Database } from './Database';
 import { DatabaseManager } from './DatabaseManager';
-import { DatabaseLocationUpdateLogStatus, DbLocationsRow } from './types';
+import {
+  DatabaseLocationUpdateLogStatus,
+  DbLocationsRow,
+  DbLocationsRowWithStatus,
+} from './types';
 import { EntityPolicy, Entity } from '@backstage/catalog-model';
 import { IngestionModel } from '..';
 
@@ -43,10 +47,13 @@ describe('DatabaseManager', () => {
     });
 
     it('can update a single location', async () => {
-      const location: DbLocationsRow = {
+      const location: DbLocationsRowWithStatus = {
         id: '123',
         type: 'some',
         target: 'thing',
+        message: '',
+        status: DatabaseLocationUpdateLogStatus.SUCCESS,
+        timestamp: new Date(314159265).toISOString(),
       };
       const desc: Entity = {
         apiVersion: 'backstage.io/v1beta1',
@@ -54,6 +61,7 @@ describe('DatabaseManager', () => {
         metadata: { name: 'c1' },
         spec: { type: 'service' },
       };
+
       const tx = (undefined as unknown) as Knex.Transaction<any, any>;
 
       const db = ({

plugins/catalog-backend/src/database/migrations/20200527114117_location_update_log_latest_view.ts

@@ -0,0 +1,35 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import * as Knex from 'knex';
+
+export async function up(knex: Knex): Promise<any> {
+  // Need to first order by date of creation
+  const query = knex
+    .select()
+    .from('location_update_log')
+    .orderBy('location_update_log.created_at', 'desc');
+
+  // And only then to do the grouping to get the latest per location
+  const groupedQuery = knex(query).groupBy('location_id').select();
+
+  await knex.schema.raw(
+    `CREATE VIEW location_update_log_latest AS ${groupedQuery.toString()};`,
+  );
+}
+
+export async function down(knex: Knex): Promise<any> {
+  return knex.schema.raw(`DROP VIEW location_update_log_latest;`);
+}

plugins/catalog-backend/src/database/types.ts

@@ -52,6 +52,12 @@ export type DbLocationsRow = {
   target: string;
 };
 
+export type DbLocationsRowWithStatus = DbLocationsRow & {
+  status: string | null;
+  timestamp: string | null;
+  message: string | null;
+};
+
 export type AddDatabaseLocation = {
   type: string;
   target: string;

plugins/catalog-backend/src/service/router.test.ts

@@ -32,6 +32,7 @@ class MockLocationsCatalog implements LocationsCatalog {
   removeLocation = jest.fn();
   locations = jest.fn();
   location = jest.fn();
+  locationHistory = jest.fn();
 }
 
 describe('createRouter', () => {

plugins/catalog-backend/src/service/router.ts

@@ -22,7 +22,7 @@ import {
   addLocationSchema,
   EntitiesCatalog,
   EntityFilters,
-  LocationsCatalog
+  LocationsCatalog,
 } from '../catalog';
 import { validateRequestBody } from './util';
 
@@ -84,6 +84,11 @@ export async function createRouter(
         const output = await locationsCatalog.locations();
         res.status(200).send(output);
       })
+      .get('/locations/:id/history', async (req, res) => {
+        const { id } = req.params;
+        const output = await locationsCatalog.locationHistory(id);
+        res.status(200).send(output);
+      })
       .get('/locations/:id', async (req, res) => {
         const { id } = req.params;
         const output = await locationsCatalog.location(id);

plugins/catalog/src/api/CatalogClient.ts

@@ -0,0 +1,45 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { CatalogApi } from './types';
+import { DescriptorEnvelope } from '../types';
+
+export class CatalogClient implements CatalogApi {
+  private apiOrigin: string;
+  private basePath: string;
+  constructor({
+    apiOrigin,
+    basePath,
+  }: {
+    apiOrigin: string;
+    basePath: string;
+  }) {
+    this.apiOrigin = apiOrigin;
+    this.basePath = basePath;
+  }
+  async getEntities(): Promise<DescriptorEnvelope[]> {
+    const response = await fetch(`${this.apiOrigin}${this.basePath}/entities`);
+    return await response.json();
+  }
+  async getEntityByName(name: string): Promise<DescriptorEnvelope> {
+    const response = await fetch(
+      `${this.apiOrigin}${this.basePath}/entities/by-name/Component/default/${name}`,
+    );
+    const entity = await response.json();
+    if (entity) return entity;
+    throw new Error(`'Entity not found: ${name}`);
+  }
+}

plugins/catalog/src/types.ts

@@ -0,0 +1,159 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export interface ComponentDescriptorV1beta1 extends DescriptorEnvelope {
+  spec: {
+    type: string;
+  };
+}
+
+export type ComponentDescriptor = ComponentDescriptorV1beta1;
+
+/**
+ * Metadata fields common to all versions/kinds of entity.
+ *
+ * @see https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#objectmeta-v1-meta
+ */
+export type EntityMeta = {
+  /**
+   * A globally unique ID for the entity.
+   *
+   * This field can not be set by the user at creation time, and the server
+   * will reject an attempt to do so. The field will be populated in read
+   * operations. The field can (optionally) be specified when performing
+   * update or delete operations, but the server is free to reject requests
+   * that do so in such a way that it breaks semantics.
+   */
+  uid?: string;
+
+  /**
+   * An opaque string that changes for each update operation to any part of
+   * the entity, including metadata.
+   *
+   * This field can not be set by the user at creation time, and the server
+   * will reject an attempt to do so. The field will be populated in read
+   * operations. The field can (optionally) be specified when performing
+   * update or delete operations, and the server will then reject the
+   * operation if it does not match the current stored value.
+   */
+  etag?: string;
+
+  /**
+   * A positive nonzero number that indicates the current generation of data
+   * for this entity; the value is incremented each time the spec changes.
+   *
+   * This field can not be set by the user at creation time, and the server
+   * will reject an attempt to do so. The field will be populated in read
+   * operations.
+   */
+  generation?: number;
+
+  /**
+   * The name of the entity.
+   *
+   * Must be uniqe within the catalog at any given point in time, for any
+   * given namespace, for any given kind.
+   */
+  name: string;
+
+  /**
+   * The short description of the entity.
+   *
+   * A a human readable string.
+   */
+  description: string;
+
+  /**
+   * The namespace that the entity belongs to.
+   */
+  namespace?: string;
+
+  /**
+   * Key/value pairs of identifying information attached to the entity.
+   */
+  labels?: Record<string, string>;
+
+  /**
+   * Key/value pairs of non-identifying auxiliary information attached to the
+   * entity.
+   */
+  annotations?: Record<string, string>;
+};
+
+/**
+ * The format envelope that's common to all versions/kinds.
+ *
+ * @see https://kubernetes.io/docs/concepts/overview/working-with-objects/kubernetes-objects/
+ */
+export type DescriptorEnvelope = {
+  /**
+   * The version of specification format for this particular entity that
+   * this is written against.
+   */
+  apiVersion: string;
+
+  /**
+   * The high level entity type being described.
+   */
+  kind: string;
+
+  /**
+   * Optional metadata related to the entity.
+   */
+  metadata: EntityMeta;
+
+  /**
+   * The specification data describing the entity itself.
+   */
+  spec?: object;
+};
+
+/**
+ * Parses and validates descriptors.
+ *
+ * The output must be validated and well formed.
+ */
+export type DescriptorParser = {
+  /**
+   * Parses and validates a single raw descriptor.
+   *
+   * @param descriptor A raw descriptor object
+   * @returns A structure describing the parsed and validated descriptor
+   * @throws An Error if the descriptor was malformed
+   */
+  parse(descriptor: object): Promise<DescriptorEnvelope>;
+};
+
+/**
+ * Parses and validates a single envelope into its materialized kind.
+ *
+ * These parsers may assume that the envelope is already validated and well
+ * formed.
+ */
+export type KindParser = {
+  /**
+   * Try to parse an envelope into a materialized kind.
+   *
+   * @param envelope A valid descriptor envelope
+   * @returns A materialized type, or undefined if the given version/kind is
+   *          not meant to be handled by this parser
+   * @throws An Error if the type was handled and found to not be properly
+   *         formatted
+   */
+  tryParse(
+    envelope: DescriptorEnvelope,
+  ): Promise<DescriptorEnvelope | undefined>;
+};