Merge branch 'master' into timbonicus/example-data

plugins/auth-backend/src/service/router.ts

@@ -17,7 +17,6 @@
 import express from 'express';
 import Router from 'express-promise-router';
 import cookieParser from 'cookie-parser';
-import Knex from 'knex';
 import { Logger } from 'winston';
 import { createAuthProvider } from '../providers';
 import { Config } from '@backstage/config';
@@ -25,11 +24,12 @@ import { DatabaseKeyStore, TokenFactory, createOidcRouter } from '../identity';
 import {
   NotFoundError,
   PluginEndpointDiscovery,
+  PluginDatabaseManager,
 } from '@backstage/backend-common';
 
 export interface RouterOptions {
   logger: Logger;
-  database: Knex;
+  database: PluginDatabaseManager;
   config: Config;
   discovery: PluginEndpointDiscovery;
 }
@@ -47,7 +47,9 @@ export async function createRouter({
 
   const keyDurationSeconds = 3600;
 
-  const keyStore = await DatabaseKeyStore.create({ database });
+  const keyStore = await DatabaseKeyStore.create({
+    database: await database.getClient(),
+  });
   const tokenIssuer = new TokenFactory({
     issuer: authUrl,
     keyStore,

plugins/auth-backend/src/service/standaloneServer.ts

@@ -53,7 +53,11 @@ export async function startStandaloneServer(
   const router = await createRouter({
     logger,
     config,
-    database,
+    database: {
+      async getClient() {
+        return database;
+      },
+    },
     discovery,
   });
 

plugins/catalog-backend/migrations/20201007201501_index_entity_search.js

@@ -0,0 +1,37 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// @ts-check
+
+/**
+ * @param {import('knex')} knex
+ */
+exports.up = async function up(knex) {
+  await knex.schema.alterTable('entities_search', table => {
+    table.index(['key'], 'entities_search_key');
+    table.index(['value'], 'entities_search_value');
+  });
+};
+
+/**
+ * @param {import('knex')} knex
+ */
+exports.down = async function down(knex) {
+  await knex.schema.alterTable('entities_search', table => {
+    table.dropIndex('', 'entities_search_key');
+    table.dropIndex('', 'entities_search_value');
+  });
+};

plugins/catalog-backend/package.json

@@ -31,12 +31,13 @@
     "express": "^4.17.1",
     "express-promise-router": "^3.0.3",
     "fs-extra": "^9.0.0",
-    "git-url-parse": "^11.2.0",
+    "git-url-parse": "^11.3.0",
     "knex": "^0.21.1",
     "ldapjs": "^2.2.0",
     "lodash": "^4.17.15",
     "morgan": "^1.10.0",
     "node-fetch": "^2.6.0",
+    "p-limit": "^3.0.2",
     "sqlite3": "^5.0.0",
     "uuid": "^8.0.0",
     "winston": "^3.2.1",

plugins/catalog-backend/src/catalog/CoalescedEntitiesCatalog.test.ts

@@ -1,160 +0,0 @@
-/*
- * Copyright 2020 Spotify AB
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import { Entity, ENTITY_DEFAULT_NAMESPACE } from '@backstage/catalog-model';
-import { Logger } from 'winston';
-import { CoalescedEntitiesCatalog } from './CoalescedEntitiesCatalog';
-import { EntitiesCatalog } from './types';
-
-describe('CoalescedEntitiesCatalog', () => {
-  const e1: Entity = {
-    apiVersion: 'a',
-    kind: 'k',
-    metadata: { name: 'n1' },
-  };
-
-  const e2: Entity = {
-    apiVersion: 'a',
-    kind: 'k',
-    metadata: { name: 'n2' },
-  };
-
-  const c1: jest.Mocked<EntitiesCatalog> = {
-    entities: jest.fn(),
-    entityByUid: jest.fn(),
-    entityByName: jest.fn(),
-    addOrUpdateEntity: jest.fn(),
-    removeEntityByUid: jest.fn(),
-  };
-
-  const c2: jest.Mocked<EntitiesCatalog> = {
-    entities: jest.fn(),
-    entityByUid: jest.fn(),
-    entityByName: jest.fn(),
-    addOrUpdateEntity: jest.fn(),
-    removeEntityByUid: jest.fn(),
-  };
-
-  const mockLogger = {
-    warn: jest.fn(),
-  };
-  const logger = (mockLogger as unknown) as Logger;
-
-  beforeEach(() => {
-    jest.resetAllMocks();
-  });
-
-  describe('entities', () => {
-    it('flattens results from multiple sources', async () => {
-      c1.entities.mockResolvedValueOnce([e1]);
-      c2.entities.mockResolvedValueOnce([e2]);
-      const catalog = new CoalescedEntitiesCatalog([c1, c2], logger);
-      await expect(catalog.entities()).resolves.toEqual(
-        expect.arrayContaining([e1, e2]),
-      );
-      expect(c1.entities).toBeCalledTimes(1);
-      expect(c2.entities).toBeCalledTimes(1);
-    });
-
-    it('logs an error if any source throws', async () => {
-      c1.entities.mockResolvedValueOnce([e1]);
-      c2.entities.mockRejectedValueOnce(new Error('boo'));
-      const catalog = new CoalescedEntitiesCatalog([c1, c2], logger);
-      await expect(catalog.entities()).resolves.toEqual([e1]);
-      expect(c1.entities).toBeCalledTimes(1);
-      expect(c2.entities).toBeCalledTimes(1);
-      expect(mockLogger.warn).toBeCalledWith(expect.stringMatching(/boo/));
-    });
-  });
-
-  describe('entityByUid', () => {
-    it('returns the first non-undefined result', async () => {
-      c1.entityByUid.mockResolvedValueOnce(undefined);
-      c2.entityByUid.mockResolvedValueOnce(e2);
-      const catalog = new CoalescedEntitiesCatalog([c1, c2], logger);
-      await expect(catalog.entityByUid('e2')).resolves.toBe(e2);
-      expect(c1.entityByUid).toBeCalledTimes(1);
-      expect(c2.entityByUid).toBeCalledTimes(1);
-    });
-
-    it('returns undefined if all results were undefined', async () => {
-      c1.entityByUid.mockResolvedValueOnce(undefined);
-      c2.entityByUid.mockResolvedValueOnce(undefined);
-      const catalog = new CoalescedEntitiesCatalog([c1, c2], logger);
-      await expect(catalog.entityByUid('e2')).resolves.toBeUndefined();
-      expect(c1.entityByUid).toBeCalledTimes(1);
-      expect(c2.entityByUid).toBeCalledTimes(1);
-    });
-
-    it('logs an error if any source throws', async () => {
-      c1.entityByUid.mockResolvedValueOnce(e1);
-      c2.entityByUid.mockRejectedValueOnce(new Error('boo'));
-      const catalog = new CoalescedEntitiesCatalog([c1, c2], logger);
-      await expect(catalog.entityByUid('e2')).resolves.toBe(e1);
-      expect(c1.entityByUid).toBeCalledTimes(1);
-      expect(c2.entityByUid).toBeCalledTimes(1);
-      expect(mockLogger.warn).toBeCalledWith(expect.stringMatching(/boo/));
-    });
-  });
-
-  describe('entityByName', () => {
-    it('returns the first non-undefined result', async () => {
-      c1.entityByName.mockResolvedValueOnce(undefined);
-      c2.entityByName.mockResolvedValueOnce(e2);
-      const catalog = new CoalescedEntitiesCatalog([c1, c2], logger);
-      await expect(
-        catalog.entityByName({
-          kind: 'k',
-          namespace: ENTITY_DEFAULT_NAMESPACE,
-          name: 'n2',
-        }),
-      ).resolves.toBe(e2);
-      expect(c1.entityByName).toBeCalledTimes(1);
-      expect(c2.entityByName).toBeCalledTimes(1);
-    });
-
-    it('returns undefined if all results were undefined', async () => {
-      c1.entityByName.mockResolvedValueOnce(undefined);
-      c2.entityByName.mockResolvedValueOnce(undefined);
-      const catalog = new CoalescedEntitiesCatalog([c1, c2], logger);
-      await expect(
-        catalog.entityByName({
-          kind: 'k',
-          namespace: ENTITY_DEFAULT_NAMESPACE,
-          name: 'n2',
-        }),
-      ).resolves.toBeUndefined();
-      expect(c1.entityByName).toBeCalledTimes(1);
-      expect(c2.entityByName).toBeCalledTimes(1);
-    });
-
-    it('logs an error if any source throws', async () => {
-      c1.entityByName.mockResolvedValueOnce(e1);
-      c2.entityByName.mockRejectedValueOnce(new Error('boo'));
-      const catalog = new CoalescedEntitiesCatalog([c1, c2], logger);
-      await expect(
-        catalog.entityByName({
-          kind: 'k',
-          namespace: ENTITY_DEFAULT_NAMESPACE,
-          name: 'n2',
-        }),
-      ).resolves.toBe(e1);
-      expect(c1.entityByName).toBeCalledTimes(1);
-      expect(c2.entityByName).toBeCalledTimes(1);
-      expect(mockLogger.warn).toBeCalledWith(expect.stringMatching(/boo/));
-    });
-  });
-});

plugins/catalog-backend/src/catalog/CoalescedEntitiesCatalog.ts

@@ -1,96 +0,0 @@
-/*
- * Copyright 2020 Spotify AB
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import { Entity, EntityName } from '@backstage/catalog-model';
-import { Logger } from 'winston';
-import { EntityFilters } from '../database';
-import { EntitiesCatalog } from './types';
-
-/**
- * A simple coalescing catalog wrapper, that acts as a front for collecting
- * catalog data from multiple sources.
- *
- * One possible usage could be to have this as a front to both a
- * DatabaseEntitiesCatalog that holds Component kinds, and another company-
- * specific catalog that is a thin wrapper on top of LDAP that supplies Group
- * and User entities. That way you'll get a coherent view of two very different
- * entity sources.
- *
- * This is mainly meant as a functional example, and you may want to provide
- * your own more specialized collector if you have this distinct need. This
- * one does not support adding/updating entities through the API for example.
- * A more competent implementation may direct the writes to different catalogs
- * based on entity kind or similar.
- */
-export class CoalescedEntitiesCatalog implements EntitiesCatalog {
-  private inner: EntitiesCatalog[];
-  private logger: Logger;
-
-  constructor(inner: EntitiesCatalog[], logger: Logger) {
-    this.inner = inner;
-    this.logger = logger;
-  }
-
-  async entities(filters?: EntityFilters): Promise<Entity[]> {
-    const ops = this.inner.map(async catalog => {
-      try {
-        return await catalog.entities(filters);
-      } catch (e) {
-        this.logger.warn(`Inner entities call failed, ${e}`);
-        return [];
-      }
-    });
-
-    const results = await Promise.all(ops);
-    return results.flat();
-  }
-
-  async entityByUid(uid: string): Promise<Entity | undefined> {
-    const ops = this.inner.map(async catalog => {
-      try {
-        return await catalog.entityByUid(uid);
-      } catch (e) {
-        this.logger.warn(`Inner entityByUid call failed, ${e}`);
-        return undefined;
-      }
-    });
-
-    const results = await Promise.all(ops);
-    return results.find(Boolean);
-  }
-
-  async entityByName(name: EntityName): Promise<Entity | undefined> {
-    const ops = this.inner.map(async catalog => {
-      try {
-        return await catalog.entityByName(name);
-      } catch (e) {
-        this.logger.warn(`Inner entityByName call failed, ${e}`);
-        return undefined;
-      }
-    });
-
-    const results = await Promise.all(ops);
-    return results.find(Boolean);
-  }
-
-  addOrUpdateEntity(): Promise<Entity> {
-    throw new Error('Method not implemented.');
-  }
-
-  removeEntityByUid(): Promise<void> {
-    throw new Error('Method not implemented.');
-  }
-}

plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.test.ts

@@ -24,12 +24,12 @@ describe('DatabaseEntitiesCatalog', () => {
   beforeAll(() => {
     db = {
       transaction: jest.fn(),
-      addEntity: jest.fn(),
+      addEntities: jest.fn(),
       updateEntity: jest.fn(),
       entities: jest.fn(),
       entityByName: jest.fn(),
       entityByUid: jest.fn(),
-      removeEntity: jest.fn(),
+      removeEntityByUid: jest.fn(),
       addLocation: jest.fn(),
       removeLocation: jest.fn(),
       location: jest.fn(),
@@ -56,7 +56,7 @@ describe('DatabaseEntitiesCatalog', () => {
       };
 
       db.entities.mockResolvedValue([]);
-      db.addEntity.mockResolvedValue({ entity });
+      db.addEntities.mockResolvedValue([{ entity }]);
 
       const catalog = new DatabaseEntitiesCatalog(db);
       const result = await catalog.addOrUpdateEntity(entity);
@@ -67,7 +67,7 @@ describe('DatabaseEntitiesCatalog', () => {
         namespace: 'd',
         name: 'c',
       });
-      expect(db.addEntity).toHaveBeenCalledTimes(1);
+      expect(db.addEntities).toHaveBeenCalledTimes(1);
       expect(result).toBe(entity);
     });
 

plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.ts

@@ -72,13 +72,25 @@ export class DatabaseEntitiesCatalog implements EntitiesCatalog {
           existing.entity.metadata.generation,
         );
       } else {
-        response = await this.database.addEntity(tx, { locationId, entity });
+        const added = await this.database.addEntities(tx, [
+          { locationId, entity },
+        ]);
+        response = added[0];
       }
 
       return response.entity;
     });
   }
 
+  async addEntities(entities: Entity[], locationId?: string): Promise<void> {
+    await this.database.transaction(async tx => {
+      await this.database.addEntities(
+        tx,
+        entities.map(entity => ({ locationId, entity })),
+      );
+    });
+  }
+
   async removeEntityByUid(uid: string): Promise<void> {
     return await this.database.transaction(async tx => {
       const entityResponse = await this.database.entityByUid(tx, uid);
@@ -96,7 +108,10 @@ export class DatabaseEntitiesCatalog implements EntitiesCatalog {
           ])
         : [entityResponse];
       for (const dbResponse of colocatedEntities) {
-        await this.database.removeEntity(tx, dbResponse?.entity.metadata.uid!);
+        await this.database.removeEntityByUid(
+          tx,
+          dbResponse?.entity.metadata.uid!,
+        );
       }
 
       if (entityResponse.locationId) {

plugins/catalog-backend/src/catalog/StaticEntitiesCatalog.ts

@@ -1,56 +0,0 @@
-/*
- * Copyright 2020 Spotify AB
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import { Entity, EntityName, getEntityName } from '@backstage/catalog-model';
-import lodash from 'lodash';
-import type { EntitiesCatalog } from './types';
-
-export class StaticEntitiesCatalog implements EntitiesCatalog {
-  private _entities: Entity[];
-
-  constructor(entities: Entity[]) {
-    this._entities = entities;
-  }
-
-  async entities(): Promise<Entity[]> {
-    return lodash.cloneDeep(this._entities);
-  }
-
-  async entityByUid(uid: string): Promise<Entity | undefined> {
-    const item = this._entities.find(e => uid === e.metadata.uid);
-    return item ? lodash.cloneDeep(item) : undefined;
-  }
-
-  async entityByName(name: EntityName): Promise<Entity | undefined> {
-    const item = this._entities.find(e => {
-      const candidate = getEntityName(e);
-      return (
-        name.kind.toLowerCase() === candidate.kind.toLowerCase() &&
-        name.namespace.toLowerCase() === candidate.namespace.toLowerCase() &&
-        name.name.toLowerCase() === candidate.name.toLowerCase()
-      );
-    });
-    return item ? lodash.cloneDeep(item) : undefined;
-  }
-
-  async addOrUpdateEntity(): Promise<Entity> {
-    throw new Error('Not supported');
-  }
-
-  async removeEntityByUid(): Promise<void> {
-    throw new Error('Not supported');
-  }
-}

plugins/catalog-backend/src/catalog/index.ts

@@ -16,5 +16,4 @@
 
 export { DatabaseEntitiesCatalog } from './DatabaseEntitiesCatalog';
 export { DatabaseLocationsCatalog } from './DatabaseLocationsCatalog';
-export { StaticEntitiesCatalog } from './StaticEntitiesCatalog';
 export type { EntitiesCatalog, LocationsCatalog } from './types';

plugins/catalog-backend/src/catalog/types.ts

@@ -26,6 +26,7 @@ export type EntitiesCatalog = {
   entityByUid(uid: string): Promise<Entity | undefined>;
   entityByName(name: EntityName): Promise<Entity | undefined>;
   addOrUpdateEntity(entity: Entity, locationId?: string): Promise<Entity>;
+  addEntities(entities: Entity[], locationId?: string): Promise<void>;
   removeEntityByUid(uid: string): Promise<void>;
 };
 

plugins/catalog-backend/src/database/CommonDatabase.test.ts

@@ -17,12 +17,12 @@
 import { ConflictError } from '@backstage/backend-common';
 import type { Entity, Location } from '@backstage/catalog-model';
 import { DatabaseManager } from './DatabaseManager';
-import { Database, DatabaseLocationUpdateLogStatus } from './types';
 import type {
   DbEntityRequest,
   DbEntityResponse,
   DbLocationsRowWithStatus,
 } from './types';
+import { Database, DatabaseLocationUpdateLogStatus } from './types';
 
 const bootstrapLocation = {
   id: expect.any(String),
@@ -135,36 +135,99 @@ describe('CommonDatabase', () => {
     await expect(db.location(location.id)).rejects.toThrow(/Found no location/);
   });
 
-  describe('addEntity', () => {
-    it('happy path: adds entity to empty database', async () => {
-      const added = await db.transaction(tx => db.addEntity(tx, entityRequest));
-      expect(added).toStrictEqual(entityResponse);
-      expect(added.entity.metadata.generation).toBe(1);
+  describe('addEntities', () => {
+    it('happy path: adds entities to empty database', async () => {
+      const result = await db.transaction(tx =>
+        db.addEntities(tx, [entityRequest]),
+      );
+      expect(result).toEqual([entityResponse]);
     });
 
     it('rejects adding the same-named entity twice', async () => {
-      await db.transaction(tx => db.addEntity(tx, entityRequest));
+      const req: DbEntityRequest[] = [
+        {
+          entity: {
+            apiVersion: 'av1',
+            kind: 'k1',
+            metadata: { name: 'n1', namespace: 'ns1' },
+          },
+        },
+        {
+          entity: {
+            apiVersion: 'av1',
+            kind: 'k1',
+            metadata: { name: 'n1', namespace: 'ns1' },
+          },
+        },
+      ];
       await expect(
-        db.transaction(tx => db.addEntity(tx, entityRequest)),
+        db.transaction(tx => db.addEntities(tx, req)),
       ).rejects.toThrow(ConflictError);
     });
 
     it('rejects adding the almost-same-namespace entity twice', async () => {
-      entityRequest.entity.metadata.namespace = undefined;
-      await db.transaction(tx => db.addEntity(tx, entityRequest));
-      entityRequest.entity.metadata.namespace = '';
+      const req: DbEntityRequest[] = [
+        {
+          entity: {
+            apiVersion: 'av1',
+            kind: 'k1',
+            metadata: { name: 'n1', namespace: 'ns1' },
+          },
+        },
+        {
+          entity: {
+            apiVersion: 'av1',
+            kind: 'k1',
+            metadata: { name: 'n1', namespace: 'nS1' },
+          },
+        },
+      ];
       await expect(
-        db.transaction(tx => db.addEntity(tx, entityRequest)),
+        db.transaction(tx => db.addEntities(tx, req)),
       ).rejects.toThrow(ConflictError);
     });
 
     it('accepts adding the same-named entity twice if on different namespaces', async () => {
-      entityRequest.entity.metadata.namespace = 'namespace1';
-      await db.transaction(tx => db.addEntity(tx, entityRequest));
-      entityRequest.entity.metadata.namespace = 'namespace2';
+      const req: DbEntityRequest[] = [
+        {
+          entity: {
+            apiVersion: 'av1',
+            kind: 'k1',
+            metadata: { name: 'n1', namespace: 'ns1' },
+          },
+        },
+        {
+          entity: {
+            apiVersion: 'av1',
+            kind: 'k1',
+            metadata: { name: 'n1', namespace: 'ns2' },
+          },
+        },
+      ];
       await expect(
-        db.transaction(tx => db.addEntity(tx, entityRequest)),
-      ).resolves.toBeDefined();
+        db.transaction(tx => db.addEntities(tx, req)),
+      ).resolves.toEqual([
+        {
+          entity: expect.objectContaining({
+            metadata: expect.objectContaining({
+              namespace: 'ns1',
+              uid: expect.any(String),
+              etag: expect.any(String),
+              generation: expect.any(Number),
+            }),
+          }),
+        },
+        {
+          entity: expect.objectContaining({
+            metadata: expect.objectContaining({
+              namespace: 'ns2',
+              uid: expect.any(String),
+              etag: expect.any(String),
+              generation: expect.any(Number),
+            }),
+          }),
+        },
+      ]);
     });
   });
 
@@ -216,7 +279,9 @@ describe('CommonDatabase', () => {
 
   describe('updateEntity', () => {
     it('can read and no-op-update an entity', async () => {
-      const added = await db.transaction(tx => db.addEntity(tx, entityRequest));
+      const [added] = await db.transaction(tx =>
+        db.addEntities(tx, [entityRequest]),
+      );
       const updated = await db.transaction(tx =>
         db.updateEntity(tx, { entity: added.entity }),
       );
@@ -233,7 +298,9 @@ describe('CommonDatabase', () => {
     });
 
     it('can update name if uid matches', async () => {
-      const added = await db.transaction(tx => db.addEntity(tx, entityRequest));
+      const [added] = await db.transaction(tx =>
+        db.addEntities(tx, [entityRequest]),
+      );
       added.entity.metadata.name! = 'new!';
       const updated = await db.transaction(tx =>
         db.updateEntity(tx, { entity: added.entity }),
@@ -242,7 +309,9 @@ describe('CommonDatabase', () => {
     });
 
     it('fails to update an entity if etag does not match', async () => {
-      const added = await db.transaction(tx => db.addEntity(tx, entityRequest));
+      const [added] = await db.transaction(tx =>
+        db.addEntities(tx, [entityRequest]),
+      );
       await expect(
         db.transaction(tx =>
           db.updateEntity(tx, { entity: added.entity }, 'garbage'),
@@ -251,7 +320,9 @@ describe('CommonDatabase', () => {
     });
 
     it('fails to update an entity if generation does not match', async () => {
-      const added = await db.transaction(tx => db.addEntity(tx, entityRequest));
+      const [added] = await db.transaction(tx =>
+        db.addEntities(tx, [entityRequest]),
+      );
       await expect(
         db.transaction(tx =>
           db.updateEntity(tx, { entity: added.entity }, undefined, 1e20),
@@ -274,8 +345,7 @@ describe('CommonDatabase', () => {
         spec: { c: null },
       };
       await db.transaction(async tx => {
-        await db.addEntity(tx, { entity: e1 });
-        await db.addEntity(tx, { entity: e2 });
+        await db.addEntities(tx, [{ entity: e1 }, { entity: e2 }]);
       });
       const result = await db.transaction(async tx => db.entities(tx, []));
       expect(result.length).toEqual(2);
@@ -311,9 +381,10 @@ describe('CommonDatabase', () => {
       ];
 
       await db.transaction(async tx => {
-        for (const entity of entities) {
-          await db.addEntity(tx, { entity });
-        }
+        await db.addEntities(
+          tx,
+          entities.map(entity => ({ entity })),
+        );
       });
 
       await expect(
@@ -349,9 +420,10 @@ describe('CommonDatabase', () => {
       ];
 
       await db.transaction(async tx => {
-        for (const entity of entities) {
-          await db.addEntity(tx, { entity });
-        }
+        await db.addEntities(
+          tx,
+          entities.map(entity => ({ entity })),
+        );
       });
 
       const rows = await db.transaction(async tx =>
@@ -398,9 +470,10 @@ describe('CommonDatabase', () => {
       ];
 
       await db.transaction(async tx => {
-        for (const entity of entities) {
-          await db.addEntity(tx, { entity });
-        }
+        await db.addEntities(
+          tx,
+          entities.map(entity => ({ entity })),
+        );
       });
 
       const rows = await db.transaction(async tx =>
@@ -446,9 +519,10 @@ describe('CommonDatabase', () => {
       ];
 
       await db.transaction(async tx => {
-        for (const entity of entities) {
-          await db.addEntity(tx, { entity });
-        }
+        await db.addEntities(
+          tx,
+          entities.map(entity => ({ entity })),
+        );
       });
 
       const e1 = await db.transaction(async tx =>

plugins/catalog-backend/src/database/CommonDatabase.ts

@@ -45,6 +45,12 @@ import type {
   EntityFilters,
 } from './types';
 
+// The number of items that are sent per batch to the database layer, when
+// doing .batchInsert calls to knex. This needs to be low enough to not cause
+// errors in the underlying engine due to exceeding query limits, but large
+// enough to get the speed benefits.
+const BATCH_SIZE = 50;
+
 /**
  * The core database implementation.
  */
@@ -100,6 +106,52 @@ export class CommonDatabase implements Database {
     return { locationId: request.locationId, entity: newEntity };
   }
 
+  async addEntities(
+    txOpaque: unknown,
+    request: DbEntityRequest[],
+  ): Promise<DbEntityResponse[]> {
+    const tx = txOpaque as Knex.Transaction<any, any>;
+
+    const result: DbEntityResponse[] = [];
+    const entityRows: DbEntitiesRow[] = [];
+    const searchRows: DbEntitiesSearchRow[] = [];
+
+    for (const { entity, locationId } of request) {
+      if (entity.metadata.uid !== undefined) {
+        throw new InputError('May not specify uid for new entities');
+      } else if (entity.metadata.etag !== undefined) {
+        throw new InputError('May not specify etag for new entities');
+      } else if (entity.metadata.generation !== undefined) {
+        throw new InputError('May not specify generation for new entities');
+      }
+
+      const newEntity = {
+        ...entity,
+        metadata: {
+          ...entity.metadata,
+          uid: generateEntityUid(),
+          etag: generateEntityEtag(),
+          generation: 1,
+        },
+      };
+
+      result.push({ entity: newEntity, locationId });
+      entityRows.push(this.toEntityRow(locationId, newEntity));
+      searchRows.push(...buildEntitySearch(newEntity.metadata.uid, newEntity));
+    }
+
+    await tx.batchInsert('entities', entityRows, BATCH_SIZE);
+    await tx<DbEntitiesSearchRow>('entities_search')
+      .whereIn(
+        'entity_id',
+        entityRows.map(r => r.id),
+      )
+      .del();
+    await tx.batchInsert('entities_search', searchRows, BATCH_SIZE);
+
+    return result;
+  }
+
   async updateEntity(
     txOpaque: unknown,
     request: DbEntityRequest,
@@ -165,10 +217,10 @@ export class CommonDatabase implements Database {
   ): Promise<DbEntityResponse[]> {
     const tx = txOpaque as Knex.Transaction<any, any>;
 
-    let builder = tx<DbEntitiesRow>('entities');
-    for (const [indexU, filter] of (filters ?? []).entries()) {
-      const index = Number(indexU);
-      const key = filter.key.toLowerCase().replace(/\*/g, '%');
+    let entitiesQuery = tx<DbEntitiesRow>('entities');
+
+    for (const filter of filters || []) {
+      const key = filter.key.toLowerCase().replace(/[*]/g, '%');
       const keyOp = filter.key.includes('*') ? 'like' : '=';
 
       let matchNulls = false;
@@ -179,36 +231,54 @@ export class CommonDatabase implements Database {
         if (!value) {
           matchNulls = true;
         } else if (value.includes('*')) {
-          matchLike.push(value.toLowerCase().replace(/\*/g, '%'));
+          matchLike.push(value.toLowerCase().replace(/[*]/g, '%'));
         } else {
           matchIn.push(value.toLowerCase());
         }
       }
 
-      builder = builder
-        .leftOuterJoin(`entities_search as t${index}`, function joins() {
-          this.on('entities.id', '=', `t${index}.entity_id`);
-          this.andOn(`t${index}.key`, keyOp, tx.raw('?', [key]));
-        })
-        .where(function rules() {
-          if (matchIn.length) {
-            this.orWhereIn(`t${index}.value`, matchIn);
-          }
-          if (matchLike.length) {
-            for (const x of matchLike) {
-              this.orWhere(`t${index}.value`, 'like', tx.raw('?', [x]));
+      // NOTE(freben): This used to be a set of OUTER JOIN, which may seem to
+      // make a lot of sense. However, it had abysmal performance on sqlite
+      // when datasets grew large, so we're using IN instead.
+      const matchQuery = tx<DbEntitiesSearchRow>('entities_search')
+        .select('entity_id')
+        .where(function keyFilter() {
+          this.andWhere('key', keyOp, key);
+          this.andWhere(function valueFilter() {
+            if (matchIn.length === 1) {
+              this.orWhere({ value: matchIn[0] });
+            } else if (matchIn.length > 1) {
+              this.orWhereIn('value', matchIn);
             }
-          }
-          if (matchNulls) {
-            this.orWhereNull(`t${index}.value`);
-          }
+            if (matchLike.length) {
+              for (const x of matchLike) {
+                this.orWhere('value', 'like', tx.raw('?', [x]));
+              }
+            }
+            if (matchNulls) {
+              // Match explicit nulls, and then handle absence separately below
+              this.orWhereNull('value');
+            }
+          });
         });
+
+      // Handle absence as nulls as well
+      entitiesQuery = entitiesQuery.andWhere(function match() {
+        this.whereIn('id', matchQuery);
+        if (matchNulls) {
+          this.orWhereNotIn(
+            'id',
+            tx<DbEntitiesSearchRow>('entities_search')
+              .select('entity_id')
+              .where('key', keyOp, key),
+          );
+        }
+      });
     }
 
-    const rows = await builder
+    const rows = await entitiesQuery
       .select('entities.*')
-      .orderBy('full_name', 'asc')
-      .groupBy('id');
+      .orderBy('full_name', 'asc');
 
     return rows.map(row => this.toEntityResponse(row));
   }
@@ -249,7 +319,7 @@ export class CommonDatabase implements Database {
     return this.toEntityResponse(rows[0]);
   }
 
-  async removeEntity(txOpaque: unknown, uid: string): Promise<void> {
+  async removeEntityByUid(txOpaque: unknown, uid: string): Promise<void> {
     const tx = txOpaque as Knex.Transaction<any, any>;
 
     const result = await tx<DbEntitiesRow>('entities').where({ id: uid }).del();

plugins/catalog-backend/src/database/types.ts

@@ -89,13 +89,15 @@ export type Database = {
   transaction<T>(fn: (tx: unknown) => Promise<T>): Promise<T>;
 
   /**
-   * Adds a new entity to the catalog.
+   * Adds a set of new entities to the catalog.
    *
    * @param tx An ongoing transaction
-   * @param request The entity being added
-   * @returns The added entity, with uid, etag and generation set
+   * @param request The entities being added
    */
-  addEntity(tx: unknown, request: DbEntityRequest): Promise<DbEntityResponse>;
+  addEntities(
+    tx: unknown,
+    request: DbEntityRequest[],
+  ): Promise<DbEntityResponse[]>;
 
   /**
    * Updates an existing entity in the catalog.
@@ -132,7 +134,7 @@ export type Database = {
 
   entityByUid(tx: unknown, uid: string): Promise<DbEntityResponse | undefined>;
 
-  removeEntity(tx: unknown, uid: string): Promise<void>;
+  removeEntityByUid(tx: unknown, uid: string): Promise<void>;
 
   addLocation(location: Location): Promise<DbLocationsRow>;
 

plugins/catalog-backend/src/ingestion/HigherOrderOperations.test.ts

@@ -39,6 +39,7 @@ describe('HigherOrderOperations', () => {
       entityByUid: jest.fn(),
       entityByName: jest.fn(),
       addOrUpdateEntity: jest.fn(),
+      addEntities: jest.fn(),
       removeEntityByUid: jest.fn(),
     };
     locationsCatalog = {
@@ -191,8 +192,8 @@ describe('HigherOrderOperations', () => {
         entities: [{ entity: desc, location }],
         errors: [],
       });
-      entitiesCatalog.entityByName.mockResolvedValue(undefined);
-      entitiesCatalog.addOrUpdateEntity.mockResolvedValue(desc);
+      entitiesCatalog.entities.mockResolvedValue([]);
+      entitiesCatalog.addEntities.mockResolvedValue(undefined);
 
       await expect(
         higherOrderOperation.refreshAllLocations(),
@@ -204,18 +205,19 @@ describe('HigherOrderOperations', () => {
         type: 'some',
         target: 'thing',
       });
-      expect(entitiesCatalog.entityByName).toHaveBeenCalledTimes(1);
-      expect(entitiesCatalog.entityByName).toHaveBeenNthCalledWith(1, {
-        kind: 'Component',
-        namespace: ENTITY_DEFAULT_NAMESPACE,
-        name: 'c1',
-      });
-      expect(entitiesCatalog.addOrUpdateEntity).toHaveBeenCalledTimes(1);
-      expect(entitiesCatalog.addOrUpdateEntity).toHaveBeenNthCalledWith(
+      expect(entitiesCatalog.entities).toHaveBeenCalledTimes(1);
+      expect(entitiesCatalog.entities).toHaveBeenNthCalledWith(
         1,
-        expect.objectContaining({
-          metadata: expect.objectContaining({ name: 'c1' }),
-        }),
+        expect.arrayContaining([
+          { key: 'kind', values: ['Component'] },
+          { key: 'metadata.namespace', values: [ENTITY_DEFAULT_NAMESPACE] },
+          { key: 'metadata.name', values: ['c1'] },
+        ]),
+      );
+      expect(entitiesCatalog.addEntities).toHaveBeenCalledTimes(1);
+      expect(entitiesCatalog.addEntities).toHaveBeenNthCalledWith(
+        1,
+        [expect.objectContaining({ metadata: { name: 'c1' } })],
         '123',
       );
     });
@@ -245,8 +247,8 @@ describe('HigherOrderOperations', () => {
         entities: [{ entity: desc, location }],
         errors: [],
       });
-      entitiesCatalog.entityByName.mockResolvedValue(undefined);
-      entitiesCatalog.addOrUpdateEntity.mockResolvedValue(desc);
+      entitiesCatalog.entities.mockResolvedValue([]);
+      entitiesCatalog.addEntities.mockResolvedValue(undefined);
 
       await expect(
         higherOrderOperation.refreshAllLocations(),

plugins/catalog-backend/src/ingestion/HigherOrderOperations.ts

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-import { InputError } from '@backstage/backend-common';
+import { ConflictError, InputError } from '@backstage/backend-common';
 import {
   Entity,
   entityHasChanges,
@@ -23,15 +23,37 @@ import {
   LocationSpec,
   serializeEntityRef,
 } from '@backstage/catalog-model';
+import { chunk, groupBy } from 'lodash';
+import limiterFactory from 'p-limit';
 import { v4 as uuidv4 } from 'uuid';
 import { Logger } from 'winston';
 import { EntitiesCatalog, LocationsCatalog } from '../catalog';
+import { durationText } from '../util/timing';
 import {
   AddLocationResult,
   HigherOrderOperation,
   LocationReader,
 } from './types';
 
+type BatchContext = {
+  kind: string;
+  namespace: string;
+  location: Location;
+};
+
+// Some locations return tens or hundreds of thousands of entities. To make
+// those payloads more manageable, we break work apart in batches of this
+// many entities and write them to storage per batch.
+const BATCH_SIZE = 100;
+
+// When writing large batches, there's an increasing chance of contention in
+// the form of conflicts where we compete with other writes. Each batch gets
+// this many attempts at being written before giving up.
+const BATCH_ATTEMPTS = 3;
+
+// The number of batches that may be ongoing at the same time.
+const BATCH_CONCURRENCY = 3;
+
 /**
  * Placeholder for operations that span several catalogs and/or stretches out
  * in time.
@@ -88,7 +110,7 @@ export class HigherOrderOperations implements HigherOrderOperation {
     if (readerOutput.errors.length) {
       const item = readerOutput.errors[0];
       throw new InputError(
-        `Failed to read location ${item.location.type} ${item.location.target}, ${item.error}`,
+        `Failed to read location ${item.location.type}:${item.location.target}, ${item.error}`,
       );
     }
 
@@ -121,85 +143,209 @@ export class HigherOrderOperations implements HigherOrderOperation {
    * without changes.
    */
   async refreshAllLocations(): Promise<void> {
-    const startTimestamp = new Date().valueOf();
+    const startTimestamp = process.hrtime();
     this.logger.info('Beginning locations refresh');
 
     const locations = await this.locationsCatalog.locations();
     this.logger.info(`Visiting ${locations.length} locations`);
 
     for (const { data: location } of locations) {
-      this.logger.debug(
-        `Refreshing location id="${location.id}" type="${location.type}" target="${location.target}"`,
+      this.logger.info(
+        `Refreshing location ${location.type}:${location.target}`,
       );
       try {
         await this.refreshSingleLocation(location);
         await this.locationsCatalog.logUpdateSuccess(location.id, undefined);
       } catch (e) {
-        this.logger.debug(
-          `Failed to refresh location id="${location.id}" type="${location.type}" target="${location.target}", ${e}`,
+        this.logger.warn(
+          `Failed to refresh location ${location.type}:${location.target}, ${e}`,
         );
         await this.locationsCatalog.logUpdateFailure(location.id, e);
       }
     }
 
-    const endTimestamp = new Date().valueOf();
-    const duration = ((endTimestamp - startTimestamp) / 1000).toFixed(1);
-    this.logger.debug(`Completed locations refresh in ${duration} seconds`);
+    this.logger.info(
+      `Completed locations refresh in ${durationText(startTimestamp)}`,
+    );
   }
 
   // Performs a full refresh of a single location
   private async refreshSingleLocation(location: Location) {
+    let startTimestamp = process.hrtime();
+
     const readerOutput = await this.locationReader.read({
       type: location.type,
       target: location.target,
     });
 
     for (const item of readerOutput.errors) {
-      this.logger.debug(
-        `Failed item in location type="${item.location.type}" target="${item.location.target}", ${item.error}`,
+      this.logger.warn(
+        `Failed item in location ${item.location.type}:${item.location.target}, ${item.error}`,
       );
     }
 
     this.logger.info(
-      `Read ${readerOutput.entities.length} entities from location ${location.type} ${location.target}`,
+      `Read ${readerOutput.entities.length} entities from location ${
+        location.type
+      }:${location.target} in ${durationText(startTimestamp)}`,
     );
 
-    const startTimestamp = process.hrtime();
-    for (const item of readerOutput.entities) {
-      const { entity } = item;
+    startTimestamp = process.hrtime();
 
-      try {
-        const previous = await this.entitiesCatalog.entityByName(
-          getEntityName(entity),
-        );
+    await this.batchAddOrUpdateEntities(
+      readerOutput.entities.map(e => e.entity),
+      location,
+    );
 
-        if (!previous) {
-          await this.entitiesCatalog.addOrUpdateEntity(entity, location.id);
-        } else if (entityHasChanges(previous, entity)) {
-          await this.entitiesCatalog.addOrUpdateEntity(entity, location.id);
-        }
+    this.logger.info(
+      `Wrote ${readerOutput.entities.length} entities from location ${
+        location.type
+      }:${location.target} in ${durationText(startTimestamp)}`,
+    );
+  }
 
-        await this.locationsCatalog.logUpdateSuccess(
-          location.id,
-          entity.metadata.name,
-        );
-      } catch (error) {
-        this.logger.info(
-          `Failed refresh of entity ${serializeEntityRef(entity)}, ${error}`,
-        );
+  /**
+   * Writes a number of entities efficiently to storage.
+   *
+   * @param entities Some entities
+   * @param location The location that they all belong to
+   */
+  async batchAddOrUpdateEntities(entities: Entity[], location: Location) {
+    // Group the entities by unique kind+namespace combinations
+    const entitiesByKindAndNamespace = groupBy(entities, entity => {
+      const name = getEntityName(entity);
+      return `${name.kind}:${name.namespace}`.toLowerCase();
+    });
 
-        await this.locationsCatalog.logUpdateFailure(
-          location.id,
-          error,
-          entity.metadata.name,
+    const limiter = limiterFactory(BATCH_CONCURRENCY);
+    const tasks: Promise<void>[] = [];
+
+    for (const groupEntities of Object.values(entitiesByKindAndNamespace)) {
+      const { kind, namespace } = getEntityName(groupEntities[0]);
+
+      // Go through the new entities in reasonable chunk sizes (sometimes,
+      // sources produce tens of thousands of entities, and those are too large
+      // batch sizes to reasonably send to the database)
+      for (const batch of chunk(groupEntities, BATCH_SIZE)) {
+        tasks.push(
+          limiter(async () => {
+            const first = serializeEntityRef(batch[0]);
+            const last = serializeEntityRef(batch[batch.length - 1]);
+            this.logger.debug(
+              `Considering batch ${first}-${last} (${batch.length} entries)`,
+            );
+
+            // Retry the batch write a few times to deal with contention
+            const context = { kind, namespace, location };
+            for (let attempt = 1; attempt <= BATCH_ATTEMPTS; ++attempt) {
+              try {
+                const { toAdd, toUpdate } = await this.analyzeBatch(
+                  batch,
+                  context,
+                );
+                if (toAdd.length) await this.batchAdd(toAdd, context);
+                if (toUpdate.length) await this.batchUpdate(toUpdate, context);
+                break;
+              } catch (e) {
+                if (e instanceof ConflictError && attempt < BATCH_ATTEMPTS) {
+                  this.logger.warn(
+                    `Failed to write batch at attempt ${attempt}/${BATCH_ATTEMPTS}, ${e}`,
+                  );
+                } else {
+                  throw e;
+                }
+              }
+            }
+          }),
         );
       }
     }
 
-    const delta = process.hrtime(startTimestamp);
-    const durationMs = ((delta[0] * 1e9 + delta[1]) / 1e9).toFixed(1);
-    this.logger.info(
-      `Wrote ${readerOutput.entities.length} entities from location ${location.type} ${location.target} in ${durationMs} seconds`,
+    await Promise.all(tasks);
+  }
+
+  // Given a batch of entities that were just read from a location, take them
+  // into consideration by comparing against the existing catalog entities and
+  // produce the list of entities to be added, and the list of entities to be
+  // updated
+  private async analyzeBatch(
+    newEntities: Entity[],
+    { kind, namespace }: BatchContext,
+  ): Promise<{
+    toAdd: Entity[];
+    toUpdate: Entity[];
+  }> {
+    const markTimestamp = process.hrtime();
+
+    const names = newEntities.map(e => e.metadata.name);
+    const oldEntities = await this.entitiesCatalog.entities([
+      { key: 'kind', values: [kind] },
+      { key: 'metadata.namespace', values: [namespace] },
+      { key: 'metadata.name', values: names },
+    ]);
+
+    const oldEntitiesByName = new Map(
+      oldEntities.map(e => [e.metadata.name, e]),
+    );
+
+    const toAdd: Entity[] = [];
+    const toUpdate: Entity[] = [];
+
+    for (const newEntity of newEntities) {
+      const oldEntity = oldEntitiesByName.get(newEntity.metadata.name);
+      if (!oldEntity) {
+        toAdd.push(newEntity);
+      } else if (entityHasChanges(oldEntity, newEntity)) {
+        toUpdate.push(newEntity);
+      }
+    }
+
+    this.logger.debug(
+      `Found ${toAdd.length} entities to add, ${
+        toUpdate.length
+      } entities to update in ${durationText(markTimestamp)}`,
+    );
+
+    return { toAdd, toUpdate };
+  }
+
+  // Efficiently adds the given entities to storage, under the assumption that
+  // they do not conflict with any existing entities
+  private async batchAdd(entities: Entity[], { location }: BatchContext) {
+    const markTimestamp = process.hrtime();
+
+    await this.entitiesCatalog.addEntities(entities, location.id);
+
+    // TODO(freben): Still not batched
+    for (const entity of entities) {
+      await this.locationsCatalog.logUpdateSuccess(
+        location.id,
+        entity.metadata.name,
+      );
+    }
+
+    this.logger.debug(
+      `Added ${entities.length} entities in ${durationText(markTimestamp)}`,
+    );
+  }
+
+  // Efficiently updates the given entities into storage, under the assumption
+  // that there already exist entities with the same names
+  private async batchUpdate(entities: Entity[], { location }: BatchContext) {
+    const markTimestamp = process.hrtime();
+
+    // TODO(freben): Still not batched
+    for (const entity of entities) {
+      await this.entitiesCatalog.addOrUpdateEntity(entity);
+
+      await this.locationsCatalog.logUpdateSuccess(
+        location.id,
+        entity.metadata.name,
+      );
+    }
+
+    this.logger.debug(
+      `Updated ${entities.length} entities in ${durationText(markTimestamp)}`,
     );
   }
 }

plugins/catalog-backend/src/service/router.test.ts

@@ -35,6 +35,7 @@ describe('createRouter', () => {
       entityByUid: jest.fn(),
       entityByName: jest.fn(),
       addOrUpdateEntity: jest.fn(),
+      addEntities: jest.fn(),
       removeEntityByUid: jest.fn(),
     };
     locationsCatalog = {

plugins/catalog-backend/src/util/timing.ts

@@ -0,0 +1,31 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Returns a string with the elapsed time since the start of an operation,
+ * with some human friendly precision, e.g. "133ms" or "14.5s".
+ *
+ * @param startTimestamp The timestamp (from process.hrtime()) at the start ot
+ *                       the operation
+ */
+export function durationText(startTimestamp: [number, number]): string {
+  const delta = process.hrtime(startTimestamp);
+  const seconds = delta[0] + delta[1] / 1e9;
+  if (seconds > 1) {
+    return `${seconds.toFixed(1)}s`;
+  }
+  return `${(seconds * 1000).toFixed(0)}ms`;
+}

plugins/circleci/src/components/Router.tsx

@@ -21,16 +21,14 @@ import { BuildWithStepsPage } from './BuildWithStepsPage/';
 import { BuildsPage } from './BuildsPage';
 import { CIRCLECI_ANNOTATION } from '../constants';
 import { Entity } from '@backstage/catalog-model';
-import { WarningPanel } from '@backstage/core';
+import { MissingAnnotationEmptyState } from '@backstage/core';
 
 export const isPluginApplicableToEntity = (entity: Entity) =>
   Boolean(entity.metadata.annotations?.[CIRCLECI_ANNOTATION]);
 
 export const Router = ({ entity }: { entity: Entity }) =>
   !isPluginApplicableToEntity(entity) ? (
-    <WarningPanel title="CircleCI plugin:">
-      <pre>{CIRCLECI_ANNOTATION}</pre> annotation is missing on the entity.
-    </WarningPanel>
+    <MissingAnnotationEmptyState annotation={CIRCLECI_ANNOTATION} />
   ) : (
     <Routes>
       <Route path={`/${circleCIRouteRef.path}`} element={<BuildsPage />} />

plugins/cloudbuild/src/components/Router.tsx

@@ -20,7 +20,7 @@ import { rootRouteRef, buildRouteRef } from '../plugin';
 import { WorkflowRunDetails } from './WorkflowRunDetails';
 import { WorkflowRunsTable } from './WorkflowRunsTable';
 import { CLOUDBUILD_ANNOTATION } from './useProjectName';
-import { WarningPanel } from '@backstage/core';
+import { MissingAnnotationEmptyState } from '@backstage/core';
 
 export const isPluginApplicableToEntity = (entity: Entity) =>
   Boolean(entity.metadata.annotations?.[CLOUDBUILD_ANNOTATION]);
@@ -28,9 +28,7 @@ export const isPluginApplicableToEntity = (entity: Entity) =>
 export const Router = ({ entity }: { entity: Entity }) =>
   // TODO(shmidt-i): move warning to a separate standardized component
   !isPluginApplicableToEntity(entity) ? (
-    <WarningPanel title="Cloudbuild plugin:">
-      <pre>{CLOUDBUILD_ANNOTATION}</pre> annotation is missing on the entity.
-    </WarningPanel>
+    <MissingAnnotationEmptyState annotation={CLOUDBUILD_ANNOTATION} />
   ) : (
     <Routes>
       <Route

plugins/cost-insights/README.md

@@ -104,3 +104,11 @@ costInsights:
     metricC:
       name: Metric C
 ```
+
+## Alerts
+
+The CostInsightsApi `getAlerts` method may return any type of alert or recommendation (called collectively "Action Items" in Cost Insights) that implements the [Alert type](https://github.com/spotify/backstage/blob/master/plugins/cost-insights/src/types/Alert.tsx). This allows you to deliver any alerts or recommendations specific to your infrastructure or company migrations.
+
+The Alert type includes an `element` field to supply the JSX Element that will be rendered in the Cost Insights "Action Items" section; we recommend using Backstage's [InfoCard](https://backstage.io/storybook/?path=/story/layout-information-card--default) and [Recharts](http://recharts.org/en-US/) to show actionable visualizations.
+
+The Alert `url` should link to documentation or instructions for resolving the alert.

plugins/cost-insights/package.json

@@ -29,6 +29,8 @@
     "@material-ui/icons": "^4.9.1",
     "@material-ui/lab": "4.0.0-alpha.45",
     "@material-ui/styles": "^4.9.6",
+    "@types/react": "^16.9",
+    "@types/recharts": "^1.8.14",
     "canvas": "^2.6.1",
     "classnames": "^2.2.6",
     "history": "^5.0.0",
@@ -48,7 +50,6 @@
     "@testing-library/user-event": "^12.0.7",
     "@types/jest": "^26.0.7",
     "@types/node": "^12.0.0",
-    "@types/recharts": "^1.8.14",
     "jest-fetch-mock": "^3.0.3",
     "msw": "^0.20.5",
     "node-fetch": "^2.6.1"

plugins/cost-insights/src/api/CostInsightsApi.ts

@@ -15,7 +15,15 @@
  */
 
 import { createApiRef } from '@backstage/core';
-import { Alert, Cost, Duration, Group, Project, ProductCost } from '../types';
+import {
+  Alert,
+  Cost,
+  Duration,
+  Group,
+  Project,
+  ProductCost,
+  Maybe,
+} from '../types';
 
 export type CostInsightsApi = {
   /**
@@ -87,6 +95,9 @@ export type CostInsightsApi = {
    * in this product. The type of entity depends on the product - it may be deployed services,
    * storage buckets, managed database instances, etc.
    *
+   * If project is supplied, this should only return product costs for the given billing entity
+   * (project in GCP).
+   *
    * The time period is supplied as a Duration rather than intervals, since this is always expected
    * to return data for two bucketed time period (e.g. month vs month, or quarter vs quarter).
    *
@@ -94,11 +105,13 @@ export type CostInsightsApi = {
    * @param group
    * @param duration A time duration, such as P1M. See the Duration type for a detailed explanation
    *    of how the durations are interpreted in Cost Insights.
+   * @param project (optional) The project id from getGroupProjects or query parameters
    */
   getProductInsights(
     product: string,
     group: string,
     duration: Duration,
+    project: Maybe<string>,
   ): Promise<ProductCost>;
 
   /**

plugins/cost-insights/src/components/AlertActionCardList/AlertActionCard.test.tsx

@@ -17,19 +17,18 @@
 import React from 'react';
 import { renderInTestApp } from '@backstage/test-utils';
 import AlertActionCard from './AlertActionCard';
-import { AlertType, ProjectGrowthAlert } from '../../types';
-import { getAlertText } from '../../utils/alerts';
+import { ProjectGrowthAlert, ProjectGrowthData } from '../../types';
 import { MockScrollProvider } from '../../utils/tests';
 
-const alert = {
-  id: AlertType.ProjectGrowth,
+const data: ProjectGrowthData = {
   aggregation: [500000.8, 970502.8],
   project: 'test-project',
   periodStart: '2019-10-01',
   periodEnd: '2020-03-31',
   change: { ratio: 120, amount: 120000 },
   products: [],
-} as ProjectGrowthAlert;
+};
+const alert = new ProjectGrowthAlert(data);
 
 describe('<AlertActionCard/>', () => {
   it('Renders an alert', async () => {
@@ -40,11 +39,7 @@ describe('<AlertActionCard/>', () => {
     );
 
     expect(rendered.getByText('1')).toBeInTheDocument();
-    const text = getAlertText(alert);
-    expect(text).toBeDefined();
-    if (text) {
-      expect(rendered.getByText(text.title)).toBeInTheDocument();
-      expect(rendered.getByText(text.subtitle)).toBeInTheDocument();
-    }
+    expect(rendered.getByText(alert.title)).toBeInTheDocument();
+    expect(rendered.getByText(alert.subtitle)).toBeInTheDocument();
   });
 });

plugins/cost-insights/src/components/AlertActionCardList/AlertActionCard.tsx

@@ -17,10 +17,9 @@ import React from 'react';
 import { Avatar, Card, CardHeader } from '@material-ui/core';
 import { useScroll } from '../../hooks';
 import { Alert } from '../../types';
-import { getAlertText, getAlertNavigation } from '../../utils/alerts';
 import {
-  useAlertActionCardStyles as useStyles,
   useAlertActionCardHeader as useHeaderStyles,
+  useAlertActionCardStyles as useStyles,
 } from '../../utils/styles';
 
 type AlertActionCardProps = {
@@ -29,9 +28,8 @@ type AlertActionCardProps = {
 };
 
 const AlertActionCard = ({ alert, number }: AlertActionCardProps) => {
-  const { scrollIntoView } = useScroll(getAlertNavigation(alert, number));
+  const { scrollIntoView } = useScroll(`alert-${number}`);
   const headerClasses = useHeaderStyles();
-  const text = getAlertText(alert);
   const classes = useStyles();
 
   return (
@@ -39,8 +37,8 @@ const AlertActionCard = ({ alert, number }: AlertActionCardProps) => {
       <CardHeader
         classes={headerClasses}
         avatar={<Avatar className={classes.avatar}>{number}</Avatar>}
-        title={text?.title}
-        subheader={text?.subtitle}
+        title={alert.title}
+        subheader={alert.subtitle}
       />
     </Card>
   );

plugins/cost-insights/src/components/AlertActionCardList/AlertActionCardList.tsx

@@ -25,7 +25,7 @@ type AlertActionCardList = {
 const AlertActionCardList: FC<AlertActionCardList> = ({ alerts }) => (
   <Paper>
     {alerts.map((alert, index) => (
-      <Fragment key={`${alert.id}-${index}`}>
+      <Fragment key={`alert-${index}`}>
         <AlertActionCard alert={alert} number={index + 1} />
         {index < alerts.length - 1 && <Divider variant="fullWidth" />}
       </Fragment>

plugins/cost-insights/src/components/AlertInsights/AlertInsights.tsx

@@ -19,7 +19,6 @@ import { Grid } from '@material-ui/core';
 import AlertInsightsSection from './AlertInsightsSection';
 import AlertInsightsHeader from './AlertInsightsHeader';
 import { Alert } from '../../types';
-import { renderAlert } from '../../utils/alerts';
 
 const title = "Your team's action items";
 const subtitle =
@@ -37,11 +36,7 @@ const AlertInsights = ({ alerts }: AlertInsightsProps) => (
     <Grid item container direction="column" spacing={4}>
       {alerts.map((alert, index) => (
         <Grid item key={`alert-card-${index}`}>
-          <AlertInsightsSection
-            alert={alert}
-            number={index + 1}
-            render={renderAlert}
-          />
+          <AlertInsightsSection alert={alert} number={index + 1} />
         </Grid>
       ))}
     </Grid>

plugins/cost-insights/src/components/AlertInsights/AlertInsightsSection.tsx

@@ -16,45 +16,28 @@
 import React from 'react';
 import { Box, Button } from '@material-ui/core';
 import AlertInsightsSectionHeader from './AlertInsightsSectionHeader';
-import {
-  getAlertButtonText,
-  getAlertText,
-  getAlertUrl,
-} from '../../utils/alerts';
-import { Alert, Currency } from '../../types';
-import { useCurrency } from '../../hooks';
+import { Alert } from '../../types';
 
 type AlertInsightsSectionProps = {
   alert: Alert;
   number: number;
-  render: (alert: Alert, currency: Currency) => JSX.Element;
 };
 
-const AlertInsightsSection = ({
-  alert,
-  number,
-  render,
-}: AlertInsightsSectionProps) => {
-  const [currency] = useCurrency();
-  const text = getAlertText(alert);
-  const url = getAlertUrl(alert);
-  const buttonText = getAlertButtonText(alert);
-
+const AlertInsightsSection = ({ alert, number }: AlertInsightsSectionProps) => {
   return (
     <Box display="flex" flexDirection="column">
       <AlertInsightsSectionHeader
-        alert={alert}
-        title={text.title}
-        subtitle={text.subtitle}
+        title={alert.title}
+        subtitle={alert.subtitle}
         number={number}
       />
       <Box textAlign="left" mt={0} mb={4}>
-        <Button variant="contained" color="primary" href={url}>
-          {buttonText}
+        <Button variant="contained" color="primary" href={alert.url}>
+          {alert.buttonText || 'View Instructions'}
         </Button>
         {/* <Button color="primary">Dismiss notification</Button> */}
       </Box>
-      {render(alert, currency)}
+      {alert.element}
     </Box>
   );
 };

plugins/cost-insights/src/components/AlertInsights/AlertInsightsSectionHeader.tsx

@@ -15,26 +15,22 @@
  */
 
 import React from 'react';
-import { Avatar, Box, Typography, Grid } from '@material-ui/core';
-import { Alert } from '../../types';
-import { getAlertNavigation } from '../../utils/alerts';
+import { Avatar, Box, Grid, Typography } from '@material-ui/core';
 import { useAlertInsightsSectionStyles as useStyles } from '../../utils/styles';
 import { useScroll } from '../../hooks';
 
 type AlertInsightsSectionHeaderProps = {
-  alert: Alert;
   number: number;
   title: string;
   subtitle: string;
 };
 
 const AlertInsightsSectionHeader = ({
-  alert,
   number,
   title,
   subtitle,
 }: AlertInsightsSectionHeaderProps) => {
-  const { ScrollAnchor } = useScroll(getAlertNavigation(alert, number));
+  const { ScrollAnchor } = useScroll(`alert-${number}`);
   const classes = useStyles();
   return (
     <Box position="relative" mb={3} textAlign="left">

plugins/cost-insights/src/components/AlertInstructionsLayout/AlertInstructionsLayout.tsx

@@ -19,6 +19,7 @@ import { Box, Button, Container, makeStyles } from '@material-ui/core';
 import ChevronLeftIcon from '@material-ui/icons/ChevronLeft';
 import { Header, Page, pageTheme } from '@backstage/core';
 import { CostInsightsThemeProvider } from '../CostInsightsPage/CostInsightsThemeProvider';
+import { ConfigProvider, CurrencyProvider } from '../../hooks';
 
 const useStyles = makeStyles(theme => ({
   root: {
@@ -39,21 +40,29 @@ const AlertInstructionsLayout = ({
   const classes = useStyles();
   return (
     <CostInsightsThemeProvider>
-      <Page theme={pageTheme.tool}>
-        <Header title="Cost Insights" pageTitleOverride={title} type="Tool" />
-        <Container maxWidth="md" disableGutters className={classes.root}>
-          <Box mb={3}>
-            <Button
-              variant="outlined"
-              startIcon={<ChevronLeftIcon />}
-              href="/cost-insights"
-            >
-              Back to Cost Insights
-            </Button>
-          </Box>
-          {children}
-        </Container>
-      </Page>
+      <ConfigProvider>
+        <CurrencyProvider>
+          <Page theme={pageTheme.tool}>
+            <Header
+              title="Cost Insights"
+              pageTitleOverride={title}
+              type="Tool"
+            />
+            <Container maxWidth="md" disableGutters className={classes.root}>
+              <Box mb={3}>
+                <Button
+                  variant="outlined"
+                  startIcon={<ChevronLeftIcon />}
+                  href="/cost-insights"
+                >
+                  Back to Cost Insights
+                </Button>
+              </Box>
+              {children}
+            </Container>
+          </Page>
+        </CurrencyProvider>
+      </ConfigProvider>
     </CostInsightsThemeProvider>
   );
 };

plugins/cost-insights/src/components/CostInsightsPage/CostInsightsPage.tsx

@@ -15,7 +15,7 @@
  */
 
 import React, { useCallback, useEffect, useState } from 'react';
-import { Box, Container, Divider, Grid } from '@material-ui/core';
+import { Box, Container, Divider, Grid, Typography } from '@material-ui/core';
 import { Progress, useApi, featureFlagsApiRef } from '@backstage/core';
 import { default as MaterialAlert } from '@material-ui/lab/Alert';
 import { costInsightsApiRef } from '../../api';
@@ -41,6 +41,7 @@ import {
 } from '../../hooks';
 import { Alert, Cost, intervalsOf, Maybe, Project } from '../../types';
 import { mapLoadingToProps } from './selector';
+import ProjectSelect from '../ProjectSelect';
 
 const CostInsightsPage = () => {
   const flags = useApi(featureFlagsApiRef).getFlags();
@@ -55,7 +56,7 @@ const CostInsightsPage = () => {
   const [alerts, setAlerts] = useState<Maybe<Alert[]>>(null);
   const [error, setError] = useState<Maybe<Error>>(null);
 
-  const { pageFilters } = useFilters(p => p);
+  const { pageFilters, setPageFilters } = useFilters(p => p);
   const {
     loadingActions,
     loadingGroups,
@@ -63,6 +64,7 @@ const CostInsightsPage = () => {
     dispatchInitial,
     dispatchInsights,
     dispatchNone,
+    dispatchReset,
   } = useLoading(mapLoadingToProps);
 
   /* eslint-disable react-hooks/exhaustive-deps */
@@ -75,8 +77,15 @@ const CostInsightsPage = () => {
   const dispatchLoadingInitial = useCallback(dispatchInitial, []);
   const dispatchLoadingInsights = useCallback(dispatchInsights, []);
   const dispatchLoadingNone = useCallback(dispatchNone, []);
+  const dispatchLoadingReset = useCallback(dispatchReset, []);
   /* eslint-enable react-hooks/exhaustive-deps */
 
+  const setProject = (project: Maybe<string>) =>
+    setPageFilters({
+      ...pageFilters,
+      project: project === 'all' ? null : project,
+    });
+
   useEffect(() => {
     async function getInsights() {
       setError(null);
@@ -165,6 +174,41 @@ const CostInsightsPage = () => {
     );
   }
 
+  const onProjectSelect = (project: Maybe<string>) => {
+    setProject(project);
+    dispatchLoadingReset(loadingActions);
+  };
+
+  const CostOverviewBanner = () => (
+    <Box
+      px={3}
+      marginTop={10}
+      display="flex"
+      flexDirection="row"
+      justifyContent="space-between"
+    >
+      <Box minHeight={40} width="75%" pt={2}>
+        <Typography variant="h4">Cost Overview</Typography>
+      </Box>
+      <Box minHeight={40} maxHeight={60} display="flex">
+        {!!flags.get('cost-insights-currencies') && (
+          <Box mr={1}>
+            <CurrencySelect
+              currency={currency}
+              currencies={currencies}
+              onSelect={setCurrency}
+            />
+          </Box>
+        )}
+        <ProjectSelect
+          project={pageFilters.project}
+          projects={projects || []}
+          onSelect={onProjectSelect}
+        />
+      </Box>
+    </Box>
+  );
+
   return (
     <CostInsightsLayout groups={groups}>
       <Grid container wrap="nowrap">
@@ -180,15 +224,6 @@ const CostInsightsPage = () => {
             justifyContent="flex-end"
             mb={2}
           >
-            {!!flags.get('cost-insights-currencies') && (
-              <Box mr={1}>
-                <CurrencySelect
-                  currency={currency}
-                  currencies={currencies}
-                  onSelect={setCurrency}
-                />
-              </Box>
-            )}
             <CopyUrlToClipboard />
             <CostInsightsSupportButton />
           </Box>
@@ -212,6 +247,9 @@ const CostInsightsPage = () => {
                   <Divider />
                 </>
               )}
+              <Grid item xs>
+                <CostOverviewBanner />
+              </Grid>
               <Grid item xs>
                 <Box px={3} py={6}>
                   {!!dailyCost.aggregation.length && (

plugins/cost-insights/src/components/CostInsightsPage/selector.tsx

@@ -14,7 +14,11 @@
  * limitations under the License.
  */
 import { MapLoadingToProps } from '../../hooks';
-import { getResetState, DefaultLoadingAction } from '../../types';
+import {
+  getResetState,
+  DefaultLoadingAction,
+  getResetStateWithoutInitial,
+} from '../../types';
 
 type CostInsightsPageLoadingProps = {
   loadingActions: Array<string>;
@@ -23,6 +27,7 @@ type CostInsightsPageLoadingProps = {
   dispatchInitial: (isLoading: boolean) => void;
   dispatchInsights: (isLoading: boolean) => void;
   dispatchNone: (loadingActions: string[]) => void;
+  dispatchReset: (loadingActions: string[]) => void;
 };
 
 export const mapLoadingToProps: MapLoadingToProps<CostInsightsPageLoadingProps> = ({
@@ -39,4 +44,6 @@ export const mapLoadingToProps: MapLoadingToProps<CostInsightsPageLoadingProps>
     dispatch({ [DefaultLoadingAction.CostInsightsPage]: isLoading }),
   dispatchNone: (loadingActions: string[]) =>
     dispatch(getResetState(loadingActions)),
+  dispatchReset: (loadingActions: string[]) =>
+    dispatch(getResetStateWithoutInitial(loadingActions)),
 });

plugins/cost-insights/src/components/CostOverviewCard/CostOverviewCard.tsx

@@ -19,10 +19,8 @@ import { Box, Card, CardContent, Divider } from '@material-ui/core';
 import CostOverviewChart from '../CostOverviewChart';
 import CostOverviewChartLegend from '../CostOverviewChartLegend';
 import CostOverviewHeader from './CostOverviewHeader';
-import CostOverviewFooter from './CostOverviewFooter';
 import MetricSelect from '../MetricSelect';
 import PeriodSelect from '../PeriodSelect';
-import ProjectSelect from '../ProjectSelect';
 import { useScroll, useFilters, useConfig } from '../../hooks';
 import { mapFiltersToProps } from './selector';
 import { DefaultNavigation } from '../../utils/navigation';
@@ -45,7 +43,6 @@ const CostOverviewCard = ({
   change,
   aggregation,
   trendline,
-  projects,
 }: CostOverviewCardProps) => {
   const { metrics } = useConfig();
   const { ScrollAnchor } = useScroll(DefaultNavigation.CostOverviewCard);
@@ -73,18 +70,13 @@ const CostOverviewCard = ({
             trendline={trendline}
           />
         </Box>
-        <CostOverviewFooter>
-          <ProjectSelect
-            project={filters.project}
-            projects={projects}
-            onSelect={setProject}
-          />
+        <Box display="flex" justifyContent="flex-end" alignItems="center">
           <MetricSelect
             metric={metric}
             metrics={metrics}
             onSelect={setMetric}
           />
-        </CostOverviewFooter>
+        </Box>
       </CardContent>
     </Card>
   );

plugins/cost-insights/src/components/ProductInsightsCard/ProductInsightsCard.tsx

@@ -39,7 +39,7 @@ const ProductInsightsCard = ({ product }: ProductInsightsCardProps) => {
   const [resource, setResource] = useState<Maybe<ProductCost>>(null);
   const [error, setError] = useState<Maybe<Error>>(null);
 
-  const { group, product: productFilter, setProduct } = useFilters(
+  const { group, product: productFilter, setProduct, project } = useFilters(
     mapFiltersToProps(product.kind),
   );
   const { loadingProduct, dispatchLoading } = useLoading(
@@ -68,6 +68,7 @@ const ProductInsightsCard = ({ product }: ProductInsightsCardProps) => {
             product.kind,
             group!,
             productFilter!.duration,
+            project,
           );
           setResource(p);
         } catch (e) {
@@ -87,6 +88,7 @@ const ProductInsightsCard = ({ product }: ProductInsightsCardProps) => {
     productFilter,
     group,
     product.kind,
+    project,
   ]);
 
   const onPeriodSelect = (duration: Duration) => {

plugins/cost-insights/src/components/ProjectGrowthAlertCard/ProjectGrowthAlertCard.test.tsx

@@ -17,7 +17,7 @@
 import React from 'react';
 import { renderInTestApp } from '@backstage/test-utils';
 import ProjectGrowthAlertCard from './ProjectGrowthAlertCard';
-import { createMockProjectGrowthAlert } from '../../utils/mockData';
+import { createMockProjectGrowthData } from '../../utils/mockData';
 import { MockCurrencyProvider, MockConfigProvider } from '../../utils/tests';
 import { AlertCost, defaultCurrencies, findAlways } from '../../types';
 
@@ -29,8 +29,8 @@ const MockAlertCosts: AlertCost[] = [
   { id: 'test-id-2', aggregation: [235, 400] },
 ];
 
-const MockProjectGrowthAlert = createMockProjectGrowthAlert(alert => ({
-  ...alert,
+const MockProjectGrowthAlert = createMockProjectGrowthData(data => ({
+  ...data,
   project: MockProject,
   products: MockAlertCosts,
 }));

plugins/cost-insights/src/components/ProjectGrowthAlertCard/ProjectGrowthAlertCard.tsx

@@ -19,11 +19,11 @@ import { Box } from '@material-ui/core';
 import { InfoCard } from '@backstage/core';
 import ResourceGrowthBarChart from '../ResourceGrowthBarChart';
 import ResourceGrowthBarChartLegend from '../ResourceGrowthBarChartLegend';
-import { Duration, ProjectGrowthAlert } from '../../types';
+import { Duration, ProjectGrowthData } from '../../types';
 import { pluralOf } from '../../utils/grammar';
 
 type ProjectGrowthAlertProps = {
-  alert: ProjectGrowthAlert;
+  alert: ProjectGrowthData;
 };
 
 const ProjectGrowthAlertCard = ({ alert }: ProjectGrowthAlertProps) => {

plugins/cost-insights/src/components/ProjectGrowthInstructionsPage/ProjectGrowthInstructionsPage.tsx

@@ -18,20 +18,19 @@ import React from 'react';
 import { Box, Typography } from '@material-ui/core';
 import { InfoCard } from '@backstage/core';
 import AlertInstructionsLayout from '../AlertInstructionsLayout';
-import ProjectGrowthAlertCard from '../ProjectGrowthAlertCard';
 import {
-  AlertType,
+  Alert,
   Duration,
   Entity,
   Product,
   ProjectGrowthAlert,
+  ProjectGrowthData,
 } from '../../types';
 import ResourceGrowthBarChartLegend from '../ResourceGrowthBarChartLegend';
 import ResourceGrowthBarChart from '../ResourceGrowthBarChart';
 
 const ProjectGrowthInstructionsPage = () => {
-  const projectGrowthAlert: ProjectGrowthAlert = {
-    id: AlertType.ProjectGrowth,
+  const alertData: ProjectGrowthData = {
     project: 'example-project',
     periodStart: 'Q1 2020',
     periodEnd: 'Q2 2020',
@@ -55,6 +54,7 @@ const ProjectGrowthInstructionsPage = () => {
       },
     ],
   };
+  const projectGrowthAlert: Alert = new ProjectGrowthAlert(alertData);
 
   const product: Product = {
     kind: 'ComputeEngine',
@@ -135,7 +135,7 @@ const ProjectGrowthInstructionsPage = () => {
           comparison of cloud products over the examined time period:
         </Typography>
         <Box mt={2} mb={2}>
-          <ProjectGrowthAlertCard alert={projectGrowthAlert} />
+          {projectGrowthAlert.element}
         </Box>
         <Typography paragraph>
           This allows you to quickly see which cloud products contributed to the

plugins/cost-insights/src/components/ProjectSelect/ProjectSelect.tsx

@@ -48,7 +48,7 @@ const ProjectSelect = ({ project, projects, onSelect }: ProjectSelectProps) => {
     <Select
       className={classes.select}
       variant="outlined"
-      value={project}
+      value={project || 'all'}
       renderValue={renderValue}
       onChange={handleOnChange}
       data-testid="project-filter-select"

plugins/cost-insights/src/components/UnlabeledDataflowAlertCard/UnlabeledDataflowAlertCard.test.tsx

@@ -17,23 +17,23 @@
 import React from 'react';
 import UnlabeledDataflowAlertCard from './UnlabeledDataflowAlertCard';
 import {
-  createMockUnlabeledDataflowAlert,
+  createMockUnlabeledDataflowData,
   createMockUnlabeledDataflowAlertProject,
 } from '../../utils/mockData';
 import { renderInTestApp } from '@backstage/test-utils';
 
-const MockUnlabeledDataflowAlertMultipleProjects = createMockUnlabeledDataflowAlert(
-  alert => ({
-    ...alert,
+const MockUnlabeledDataflowAlertMultipleProjects = createMockUnlabeledDataflowData(
+  data => ({
+    ...data,
     projects: [...Array(10)].map(() =>
       createMockUnlabeledDataflowAlertProject(),
     ),
   }),
 );
 
-const MockUnlabeledDataflowAlertSingleProject = createMockUnlabeledDataflowAlert(
-  alert => ({
-    ...alert,
+const MockUnlabeledDataflowAlertSingleProject = createMockUnlabeledDataflowData(
+  data => ({
+    ...data,
     projects: [...Array(1)].map(() =>
       createMockUnlabeledDataflowAlertProject(),
     ),

plugins/cost-insights/src/components/UnlabeledDataflowAlertCard/UnlabeledDataflowAlertCard.tsx

@@ -19,11 +19,11 @@ import { Box } from '@material-ui/core';
 import { InfoCard } from '@backstage/core';
 import UnlabeledDataflowBarChart from '../UnlabeledDataflowBarChart';
 import UnlabeledDataflowBarChartLegend from '../UnlabeledDataflowBarChartLegend';
-import { UnlabeledDataflowAlert } from '../../types';
+import { UnlabeledDataflowData } from '../../types';
 import { pluralOf } from '../../utils/grammar';
 
 type UnlabeledDataflowAlertProps = {
-  alert: UnlabeledDataflowAlert;
+  alert: UnlabeledDataflowData;
 };
 
 const UnlabeledDataflowAlertCard = ({ alert }: UnlabeledDataflowAlertProps) => {

plugins/cost-insights/src/components/index.ts

@@ -13,24 +13,6 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import React from 'react';
-import { Box } from '@material-ui/core';
 
-type CostOverviewFooterProps = {
-  children?: React.ReactNode;
-};
-
-const CostOverviewFooter = ({ children }: CostOverviewFooterProps) => (
-  <Box
-    display="flex"
-    flexDirection="row"
-    justifyContent="space-between"
-    alignItems="center"
-  >
-    {React.Children.map(children, child => (
-      <Box marginY={1}>{child}</Box>
-    ))}
-  </Box>
-);
-
-export default CostOverviewFooter;
+export { default as BarChart } from './BarChart';
+export { default as LegendItem } from './LegendItem';

plugins/cost-insights/src/index.ts

@@ -16,4 +16,5 @@
 
 export { plugin } from './plugin';
 export * from './api';
+export * from './components';
 export * from './types';

plugins/cost-insights/src/types/Alert.ts

@@ -1,77 +0,0 @@
-/*
- * Copyright 2020 Spotify AB
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import { ChangeStatistic } from './ChangeStatistic';
-import { Maybe } from './Maybe';
-
-export type Alert = ProjectGrowthAlert | UnlabeledDataflowAlert;
-
-export interface AlertProps {
-  alert: Alert;
-}
-
-export enum AlertType {
-  ProjectGrowth = 'projectGrowth',
-  UnlabeledDataflow = 'unlabeledDataflow',
-}
-
-export interface AlertCost {
-  id: string;
-  aggregation: [number, number];
-}
-
-export interface ResourceData {
-  previous: number;
-  current: number;
-  name: Maybe<string>;
-}
-
-export interface BarChartData {
-  previousFill: string;
-  currentFill: string;
-  previousName: string;
-  currentName: string;
-}
-
-export enum DataKey {
-  Previous = 'previous',
-  Current = 'current',
-  Name = 'name',
-}
-
-export interface ProjectGrowthAlert {
-  id: AlertType.ProjectGrowth;
-  project: string;
-  periodStart: string;
-  periodEnd: string;
-  aggregation: [number, number];
-  change: ChangeStatistic;
-  products: Array<AlertCost>;
-}
-
-export interface UnlabeledDataflowAlert {
-  id: AlertType.UnlabeledDataflow;
-  periodStart: string;
-  periodEnd: string;
-  projects: Array<UnlabeledDataflowAlertProject>;
-  unlabeledCost: number;
-  labeledCost: number;
-}
-
-export interface UnlabeledDataflowAlertProject {
-  id: string;
-  unlabeledCost: number;
-  labeledCost: number;
-}

plugins/cost-insights/src/types/Alert.tsx

@@ -0,0 +1,123 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import React from 'react';
+import { ChangeStatistic } from './ChangeStatistic';
+import { Maybe } from './Maybe';
+import UnlabeledDataflowAlertCard from '../components/UnlabeledDataflowAlertCard';
+import ProjectGrowthAlertCard from '../components/ProjectGrowthAlertCard';
+
+/**
+ * Generic alert type with required fields for display. The `element` field will be rendered in
+ * the Cost Insights "Action Items" section. This should use data fetched in the CostInsightsApi
+ * implementation to render an InfoCard or other visualization.
+ */
+export type Alert = {
+  title: string;
+  subtitle: string;
+  url: string;
+  buttonText?: string; // Default: View Instructions
+  element: JSX.Element;
+};
+
+export interface AlertCost {
+  id: string;
+  aggregation: [number, number];
+}
+
+export interface ResourceData {
+  previous: number;
+  current: number;
+  name: Maybe<string>;
+}
+
+export interface BarChartData {
+  previousFill: string;
+  currentFill: string;
+  previousName: string;
+  currentName: string;
+}
+
+export enum DataKey {
+  Previous = 'previous',
+  Current = 'current',
+  Name = 'name',
+}
+
+/**
+ * The alerts below are examples of Alert implementation; the CostInsightsApi permits returning
+ * any implementation of the Alert type, so adopters can create their own. The CostInsightsApi
+ * fetches alert data from the backend, then creates Alert classes with the data.
+ */
+export interface ProjectGrowthData {
+  project: string;
+  periodStart: string;
+  periodEnd: string;
+  aggregation: [number, number];
+  change: ChangeStatistic;
+  products: Array<AlertCost>;
+}
+
+export class ProjectGrowthAlert implements Alert {
+  data: ProjectGrowthData;
+
+  constructor(data: ProjectGrowthData) {
+    this.data = data;
+  }
+
+  get title() {
+    return `Investigate cost growth in project ${this.data.project}`;
+  }
+
+  subtitle =
+    'Cost growth outpacing business growth is unsustainable long-term.';
+  url = '/cost-insights/investigating-growth';
+
+  get element() {
+    return <ProjectGrowthAlertCard alert={this.data} />;
+  }
+}
+
+export interface UnlabeledDataflowData {
+  periodStart: string;
+  periodEnd: string;
+  projects: Array<UnlabeledDataflowAlertProject>;
+  unlabeledCost: number;
+  labeledCost: number;
+}
+
+export class UnlabeledDataflowAlert implements Alert {
+  data: UnlabeledDataflowData;
+
+  constructor(data: UnlabeledDataflowData) {
+    this.data = data;
+  }
+
+  title = 'Add labels to workflows';
+  subtitle =
+    'Labels show in billing data, enabling cost insights for each workflow.';
+  url = '/cost-insights/labeling-jobs';
+
+  get element() {
+    return <UnlabeledDataflowAlertCard alert={this.data} />;
+  }
+}
+
+export interface UnlabeledDataflowAlertProject {
+  id: string;
+  unlabeledCost: number;
+  labeledCost: number;
+}

plugins/cost-insights/src/utils/alerts.tsx

@@ -1,82 +0,0 @@
-/*
- * Copyright 2020 Spotify AB
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import React from 'react';
-import { Alert, AlertType, assertNever } from '../types';
-import ProjectGrowthAlertCard from '../components/ProjectGrowthAlertCard';
-import UnlabeledDataflowAlertCard from '../components/UnlabeledDataflowAlertCard';
-
-export function getAlertText(alert: Alert) {
-  switch (alert.id) {
-    case AlertType.ProjectGrowth:
-      return {
-        title: `Investigate cost growth in project ${alert.project}`,
-        subtitle:
-          'Cost growth outpacing business growth is unsustainable long-term.',
-      } as AlertText;
-    case AlertType.UnlabeledDataflow:
-      return {
-        title: 'Add labels to workflows',
-        subtitle:
-          'Labels show in billing data, enabling cost insights for each workflow.',
-      };
-    default:
-      return assertNever(alert);
-  }
-}
-
-export function getAlertUrl(alert: Alert) {
-  switch (alert.id) {
-    case AlertType.ProjectGrowth:
-      return '/cost-insights/investigating-growth' as AlertUrl;
-    case AlertType.UnlabeledDataflow:
-      return '/cost-insights/labeling-jobs' as AlertUrl;
-    default:
-      return assertNever(alert);
-  }
-}
-
-export function getAlertButtonText(alert: Alert) {
-  switch (alert.id) {
-    case AlertType.ProjectGrowth:
-    case AlertType.UnlabeledDataflow:
-      return 'View Instructions' as AlertButtonText;
-    default:
-      return assertNever(alert);
-  }
-}
-
-export function getAlertNavigation(alert: Alert, number: number) {
-  return `${alert.id}-${number}`;
-}
-
-export function renderAlert(alert: Alert) {
-  switch (alert.id) {
-    case AlertType.ProjectGrowth:
-      return <ProjectGrowthAlertCard alert={alert} />;
-    case AlertType.UnlabeledDataflow:
-      return <UnlabeledDataflowAlertCard alert={alert} />;
-    default:
-      return assertNever(alert);
-  }
-}
-
-export type AlertUrl = string;
-export type AlertButtonText = string;
-
-export interface AlertText {
-  title: string;
-  subtitle: string;
-}

plugins/cost-insights/src/utils/mockData.ts

@@ -15,17 +15,16 @@
  */
 
 import {
-  AlertType,
-  Entity,
-  ProjectGrowthAlert,
-  Product,
-  UnlabeledDataflowAlert,
-  UnlabeledDataflowAlertProject,
-  getDefaultState,
   DefaultLoadingAction,
   Duration,
-  ProductCost,
+  Entity,
   findAlways,
+  getDefaultState,
+  Product,
+  ProductCost,
+  ProjectGrowthData,
+  UnlabeledDataflowAlertProject,
+  UnlabeledDataflowData,
 } from '../types';
 import { Config } from '@backstage/config';
 import { ConfigApi } from '@backstage/core';
@@ -73,11 +72,10 @@ export const createMockProductCost = (
   return { ...defaultProduct };
 };
 
-export const createMockProjectGrowthAlert = (
-  callback?: mockAlertRenderer<ProjectGrowthAlert>,
-): ProjectGrowthAlert => {
-  const defaultAlert: ProjectGrowthAlert = {
-    id: AlertType.ProjectGrowth,
+export const createMockProjectGrowthData = (
+  callback?: mockAlertRenderer<ProjectGrowthData>,
+): ProjectGrowthData => {
+  const data: ProjectGrowthData = {
     project: 'test-project-growth-alert',
     periodStart: '2019-10-01',
     periodEnd: '2020-03-31',
@@ -90,17 +88,16 @@ export const createMockProjectGrowthAlert = (
   };
 
   if (typeof callback === 'function') {
-    return callback({ ...defaultAlert });
+    return callback({ ...data });
   }
 
-  return { ...defaultAlert };
+  return { ...data };
 };
 
-export const createMockUnlabeledDataflowAlert = (
-  callback?: mockAlertRenderer<UnlabeledDataflowAlert>,
-): UnlabeledDataflowAlert => {
-  const defaultAlert: UnlabeledDataflowAlert = {
-    id: AlertType.UnlabeledDataflow,
+export const createMockUnlabeledDataflowData = (
+  callback?: mockAlertRenderer<UnlabeledDataflowData>,
+): UnlabeledDataflowData => {
+  const data: UnlabeledDataflowData = {
     periodStart: '2020-05-01',
     periodEnd: '2020-06-1',
     projects: [],
@@ -109,10 +106,10 @@ export const createMockUnlabeledDataflowAlert = (
   };
 
   if (typeof callback === 'function') {
-    return callback({ ...defaultAlert });
+    return callback({ ...data });
   }
 
-  return { ...defaultAlert };
+  return { ...data };
 };
 
 export const createMockUnlabeledDataflowAlertProject = (

plugins/cost-insights/src/utils/styles.ts

@@ -385,6 +385,7 @@ export const useSelectStyles = makeStyles<BackstageTheme>(
       select: {
         minWidth: 200,
         textAlign: 'start',
+        backgroundColor: theme.palette.background.paper,
       },
       menuItem: {
         minWidth: 200,

plugins/jenkins/src/components/Router.tsx

@@ -19,7 +19,7 @@ import { buildRouteRef, rootRouteRef } from '../plugin';
 import { DetailedViewPage } from './BuildWithStepsPage/';
 import { JENKINS_ANNOTATION } from '../constants';
 import { Entity } from '@backstage/catalog-model';
-import { WarningPanel } from '@backstage/core';
+import { MissingAnnotationEmptyState } from '@backstage/core';
 import { CITable } from './BuildsPage/lib/CITable';
 
 export const isPluginApplicableToEntity = (entity: Entity) =>
@@ -27,10 +27,7 @@ export const isPluginApplicableToEntity = (entity: Entity) =>
 
 export const Router = ({ entity }: { entity: Entity }) => {
   return !isPluginApplicableToEntity(entity) ? (
-    <WarningPanel title="Jenkins plugin:">
-      <pre>entity.metadata.annotations['{JENKINS_ANNOTATION}']</pre>
-      key is missing on the entity.
-    </WarningPanel>
+    <MissingAnnotationEmptyState annotation={JENKINS_ANNOTATION} />
   ) : (
     <Routes>
       <Route path={`/${rootRouteRef.path}`} element={<CITable />} />

plugins/kubernetes/src/Router.tsx

@@ -20,7 +20,7 @@ import { Route, Routes } from 'react-router-dom';
 
 import { rootCatalogKubernetesRouteRef } from './plugin';
 import { KubernetesContent } from './components/KubernetesContent';
-import { WarningPanel } from '@backstage/core';
+import { MissingAnnotationEmptyState } from '@backstage/core';
 
 const KUBERNETES_ANNOTATION = 'backstage.io/kubernetes-id';
 
@@ -29,11 +29,7 @@ export const Router = ({ entity }: { entity: Entity }) => {
     entity.metadata.annotations?.[KUBERNETES_ANNOTATION];
 
   if (!kubernetesAnnotationValue) {
-    return (
-      <WarningPanel title="Kubernetes plugin:">
-        <pre>{KUBERNETES_ANNOTATION}</pre> annotation is missing on the entity.
-      </WarningPanel>
-    );
+    return <MissingAnnotationEmptyState annotation={KUBERNETES_ANNOTATION} />;
   }
 
   return (

plugins/lighthouse/src/Router.tsx

@@ -23,7 +23,7 @@ import CreateAudit, { CreateAuditContent } from './components/CreateAudit';
 import { Entity } from '@backstage/catalog-model';
 import { LIGHTHOUSE_WEBSITE_URL_ANNOTATION } from '../constants';
 import { AuditListForEntity } from './components/AuditList/AuditListForEntity';
-import { EmptyState } from '@backstage/core';
+import { MissingAnnotationEmptyState } from '@backstage/core';
 
 export const isPluginApplicableToEntity = (entity: Entity) =>
   Boolean(entity.metadata.annotations?.[LIGHTHOUSE_WEBSITE_URL_ANNOTATION]);
@@ -38,10 +38,8 @@ export const Router = () => (
 
 export const EmbeddedRouter = ({ entity }: { entity: Entity }) =>
   !isPluginApplicableToEntity(entity) ? (
-    <EmptyState
-      missing="field"
-      title="Your plugin is missing an annotation"
-      description={`Please add the ${LIGHTHOUSE_WEBSITE_URL_ANNOTATION} annotation`}
+    <MissingAnnotationEmptyState
+      annotation={LIGHTHOUSE_WEBSITE_URL_ANNOTATION}
     />
   ) : (
     <Routes>

plugins/rollbar/src/components/Router.tsx

@@ -17,7 +17,7 @@
 import React from 'react';
 import { Routes, Route } from 'react-router';
 import { Entity } from '@backstage/catalog-model';
-import { WarningPanel } from '@backstage/core';
+import { MissingAnnotationEmptyState } from '@backstage/core';
 import { catalogRouteRef } from '../routes';
 import { ROLLBAR_ANNOTATION } from '../constants';
 import { EntityPageRollbar } from './EntityPageRollbar/EntityPageRollbar';
@@ -31,9 +31,7 @@ type Props = {
 
 export const Router = ({ entity }: Props) =>
   !isPluginApplicableToEntity(entity) ? (
-    <WarningPanel title="Rollbar plugin:">
-      <pre>{ROLLBAR_ANNOTATION}</pre> annotation is missing on the entity.
-    </WarningPanel>
+    <MissingAnnotationEmptyState annotation={ROLLBAR_ANNOTATION} />
   ) : (
     <Routes>
       <Route

plugins/scaffolder-backend/package.json

@@ -36,7 +36,7 @@
     "express": "^4.17.1",
     "express-promise-router": "^3.0.3",
     "fs-extra": "^9.0.0",
-    "git-url-parse": "^11.2.0",
+    "git-url-parse": "^11.3.0",
     "globby": "^11.0.0",
     "helmet": "^4.0.0",
     "jsonschema": "^1.2.6",

plugins/sentry/src/components/Router.tsx

@@ -16,7 +16,7 @@
 import React from 'react';
 import { Entity } from '@backstage/catalog-model';
 import { Routes, Route } from 'react-router';
-import { WarningPanel } from '@backstage/core';
+import { MissingAnnotationEmptyState } from '@backstage/core';
 import { SentryPluginWidget } from './SentryPluginWidget/SentryPluginWidget';
 
 const SENTRY_ANNOTATION = 'sentry.io/project-slug';
@@ -25,11 +25,7 @@ export const Router = ({ entity }: { entity: Entity }) => {
   const projectId = entity.metadata.annotations?.[SENTRY_ANNOTATION];
 
   if (!projectId) {
-    return (
-      <WarningPanel title="Sentry plugin:">
-        <pre>{SENTRY_ANNOTATION}</pre> annotation is missing on the entity.
-      </WarningPanel>
-    );
+    return <MissingAnnotationEmptyState annotation={SENTRY_ANNOTATION} />;
   }
 
   return (

plugins/techdocs-backend/package.json

@@ -31,7 +31,7 @@
     "express": "^4.17.1",
     "express-promise-router": "^3.0.3",
     "fs-extra": "^9.0.1",
-    "git-url-parse": "^11.2.0",
+    "git-url-parse": "^11.3.0",
     "knex": "^0.21.1",
     "node-fetch": "^2.6.0",
     "nodegit": "^0.27.0",

plugins/techdocs-backend/src/default-branch.ts

@@ -61,6 +61,16 @@ function getGitlabApiUrl(url: string): URL {
   );
 }
 
+function getAzureApiUrl(url: string): URL {
+  const { protocol, resource, organization, owner, name } = parseGitUrl(url);
+  const apiRepoPath = '_apis/git/repositories';
+  const apiVersion = 'api-version=6.0';
+
+  return new URL(
+    `${protocol}://${resource}/${organization}/${owner}/${apiRepoPath}/${name}?${apiVersion}`,
+  );
+}
+
 function getGithubRequestOptions(config: Config): RequestInit {
   const headers: HeadersInit = {
     Accept: 'application/vnd.github.v3.raw',
@@ -99,6 +109,26 @@ function getGitlabRequestOptions(config: Config): RequestInit {
   };
 }
 
+function getAzureRequestOptions(config: Config): RequestInit {
+  const headers: HeadersInit = {};
+
+  const token =
+    config.getOptionalString('catalog.processors.azureApi.privateToken') ??
+    process.env.AZURE_TOKEN;
+
+  if (token !== '') {
+    headers.Authorization = `Basic ${Buffer.from(`:${token}`, 'utf8').toString(
+      'base64',
+    )}`;
+  }
+
+  const requestOptions: RequestInit = {
+    headers,
+  };
+
+  return requestOptions;
+}
+
 async function getGithubDefaultBranch(
   repositoryUrl: string,
   config: Config,
@@ -159,6 +189,42 @@ async function getGitlabDefaultBranch(
   }
 }
 
+async function getAzureDefaultBranch(
+  repositoryUrl: string,
+  config: Config,
+): Promise<string> {
+  const path = getAzureApiUrl(repositoryUrl).toString();
+
+  const options = getAzureRequestOptions(config);
+
+  try {
+    const urlResponse = await fetch(path, options);
+    if (!urlResponse.ok) {
+      throw new Error(
+        `Failed to load url: ${urlResponse.status} ${urlResponse.statusText}. Make sure you have permission to repository: ${repositoryUrl}`,
+      );
+    }
+    const urlResult = await urlResponse.json();
+
+    const idResponse = await fetch(urlResult.url, options);
+    if (!idResponse.ok) {
+      throw new Error(
+        `Failed to load url: ${idResponse.status} ${idResponse.statusText}. Make sure you have permission to repository: ${urlResult.repository.url}`,
+      );
+    }
+    const idResult = await idResponse.json();
+    const name = idResult.defaultBranch;
+
+    if (!name) {
+      throw new Error('Not found Azure DevOps default branch');
+    }
+
+    return name;
+  } catch (error) {
+    throw new Error(`Failed to get Azure DevOps default branch: ${error}`);
+  }
+}
+
 export const getDefaultBranch = async (
   repositoryUrl: string,
 ): Promise<string> => {
@@ -166,6 +232,7 @@ export const getDefaultBranch = async (
   const typeMapping = [
     { url: /github*/g, type: 'github' },
     { url: /gitlab*/g, type: 'gitlab' },
+    { url: /azure*/g, type: 'azure/api' },
   ];
 
   const type = typeMapping.filter(item => item.url.test(repositoryUrl))[0]
@@ -177,6 +244,8 @@ export const getDefaultBranch = async (
         return await getGithubDefaultBranch(repositoryUrl, config);
       case 'gitlab':
         return await getGitlabDefaultBranch(repositoryUrl, config);
+      case 'azure/api':
+        return await getAzureDefaultBranch(repositoryUrl, config);
 
       default:
         throw new Error('Failed to get repository type');

plugins/techdocs-backend/src/helpers.ts

@@ -76,6 +76,7 @@ export const getLocationForEntity = (
   switch (type) {
     case 'github':
     case 'gitlab':
+    case 'azure/api':
       return { type, target };
     case 'dir':
       if (path.isAbsolute(target)) return { type, target };
@@ -124,9 +125,13 @@ export const checkoutGitRepository = async (
   const user =
     process.env.GITHUB_PRIVATE_TOKEN_USER ||
     process.env.GITLAB_PRIVATE_TOKEN_USER ||
+    process.env.AZURE_PRIVATE_TOKEN_USER ||
     '';
   const token =
-    process.env.GITHUB_TOKEN || process.env.GITLAB_PRIVATE_TOKEN_USER || '';
+    process.env.GITHUB_TOKEN ||
+    process.env.GITLAB_PRIVATE_TOKEN_USER ||
+    process.env.AZURE_TOKEN ||
+    '';
 
   if (fs.existsSync(repositoryTmpPath)) {
     try {

plugins/techdocs-backend/src/service/router.ts

@@ -78,7 +78,7 @@ export async function createRouter({
     }
   });
 
-  router.get('/metadata/entity/:kind/:namespace/:name', async (req, res) => {
+  router.get('/metadata/entity/:namespace/:kind/:name', async (req, res) => {
     const baseUrl = config.getString('backend.baseUrl');
     const { kind, namespace, name } = req.params;
 
@@ -101,7 +101,7 @@ export async function createRouter({
     }
   });
 
-  router.get('/docs/:kind/:namespace/:name/*', async (req, res) => {
+  router.get('/docs/:namespace/:kind/:name/*', async (req, res) => {
     const storageUrl = config.getString('techdocs.storageUrl');
 
     const { kind, namespace, name } = req.params;

plugins/techdocs-backend/src/techdocs/stages/prepare/azure.test.ts

@@ -0,0 +1,63 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { getVoidLogger } from '@backstage/backend-common';
+import { AzurePreparer } from './azure';
+import { checkoutGitRepository } from '../../../helpers';
+
+function normalizePath(path: string) {
+  return path
+    .replace(/^[a-z]:/i, '')
+    .split('\\')
+    .join('/');
+}
+
+jest.mock('../../../helpers', () => ({
+  ...jest.requireActual<{}>('../../../helpers'),
+  checkoutGitRepository: jest.fn(() => '/tmp/backstage-repo/org/name/branch'),
+}));
+
+const createMockEntity = (annotations = {}) => {
+  return {
+    apiVersion: 'version',
+    kind: 'TestKind',
+    metadata: {
+      name: 'test-component-name',
+      annotations: {
+        ...annotations,
+      },
+    },
+  };
+};
+
+const logger = getVoidLogger();
+
+describe('Azure DevOps preparer', () => {
+  it('should prepare temp docs path from Azure DevOps repo', async () => {
+    const preparer = new AzurePreparer(logger);
+
+    const mockEntity = createMockEntity({
+      'backstage.io/techdocs-ref':
+        'azure/api:https://dev.azure.com/backstage-org/backstage-project/_git/template-repo?path=%2Ftemplate.yaml',
+    });
+
+    const tempDocsPath = await preparer.prepare(mockEntity);
+    expect(checkoutGitRepository).toHaveBeenCalledTimes(1);
+    expect(normalizePath(tempDocsPath)).toEqual(
+      '/tmp/backstage-repo/org/name/branch/template.yaml',
+    );
+  });
+});

plugins/techdocs-backend/src/techdocs/stages/prepare/azure.ts

@@ -0,0 +1,55 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import path from 'path';
+import { Entity } from '@backstage/catalog-model';
+import { InputError } from '@backstage/backend-common';
+import { PreparerBase } from './types';
+import parseGitUrl from 'git-url-parse';
+import {
+  parseReferenceAnnotation,
+  checkoutGitRepository,
+} from '../../../helpers';
+
+import { Logger } from 'winston';
+
+export class AzurePreparer implements PreparerBase {
+  private readonly logger: Logger;
+
+  constructor(logger: Logger) {
+    this.logger = logger;
+  }
+
+  async prepare(entity: Entity): Promise<string> {
+    const { type, target } = parseReferenceAnnotation(
+      'backstage.io/techdocs-ref',
+      entity,
+    );
+
+    if (type !== 'azure/api') {
+      throw new InputError(`Wrong target type: ${type}, should be 'azure/api'`);
+    }
+
+    try {
+      const repoPath = await checkoutGitRepository(target, this.logger);
+      const parsedGitLocation = parseGitUrl(target);
+
+      return path.join(repoPath, parsedGitLocation.filepath);
+    } catch (error) {
+      this.logger.debug(`Repo checkout failed with error ${error.message}`);
+      throw error;
+    }
+  }
+}

plugins/techdocs-backend/src/techdocs/stages/prepare/dir.ts

@@ -42,7 +42,8 @@ export class DirectoryPreparer implements PreparerBase {
     );
     switch (type) {
       case 'github':
-      case 'gitlab': {
+      case 'gitlab':
+      case 'azure/api': {
         const parsedGitLocation = parseGitUrl(target);
         const repoLocation = await checkoutGitRepository(target, this.logger);
 

plugins/techdocs-backend/src/techdocs/stages/prepare/index.ts

@@ -16,5 +16,6 @@
 export { DirectoryPreparer } from './dir';
 export { GithubPreparer } from './github';
 export { GitlabPreparer } from './gitlab';
+export { AzurePreparer } from './azure';
 export { Preparers } from './preparers';
 export type { PreparerBuilder, PreparerBase } from './types';

plugins/techdocs-backend/src/techdocs/stages/prepare/types.ts

@@ -30,4 +30,4 @@ export type PreparerBuilder = {
   get(entity: Entity): PreparerBase;
 };
 
-export type RemoteProtocol = 'dir' | 'github' | 'gitlab' | 'file';
+export type RemoteProtocol = 'dir' | 'github' | 'gitlab' | 'file' | 'azure/api';

plugins/techdocs-backend/src/techdocs/stages/publish/local.test.ts

@@ -53,7 +53,7 @@ describe('local publisher', () => {
 
     const resultDir = path.resolve(
       __dirname,
-      `../../../../static/docs/${mockEntity.kind}/default/${mockEntity.metadata.name}`,
+      `../../../../static/docs/default/${mockEntity.kind}/${mockEntity.metadata.name}`,
     );
 
     expect(fs.existsSync(resultDir)).toBeTruthy();

plugins/techdocs-backend/src/techdocs/stages/publish/local.ts

@@ -42,8 +42,8 @@ export class LocalPublish implements PublisherBase {
     const publishDir = resolvePackagePath(
       '@backstage/plugin-techdocs-backend',
       'static/docs',
-      entity.kind,
       entityNamespace,
+      entity.kind,
       entity.metadata.name,
     );
 

plugins/techdocs/src/EntityPageDocs.tsx

@@ -23,7 +23,7 @@ export const EntityPageDocs = ({ entity }: { entity: Entity }) => {
     <Reader
       entityId={{
         kind: entity.kind,
-        namespace: entity.metadata.namespace,
+        namespace: entity.metadata.namespace ?? 'default',
         name: entity.metadata.name,
       }}
     />

plugins/techdocs/src/Router.tsx

@@ -17,8 +17,7 @@
 import React from 'react';
 import { Entity } from '@backstage/catalog-model';
 import { Route, Routes } from 'react-router-dom';
-import { WarningPanel } from '@backstage/core';
-
+import { MissingAnnotationEmptyState } from '@backstage/core';
 import {
   rootRouteRef,
   rootDocsRouteRef,
@@ -43,15 +42,7 @@ export const EmbeddedDocsRouter = ({ entity }: { entity: Entity }) => {
   const projectId = entity.metadata.annotations?.[TECHDOCS_ANNOTATION];
 
   if (!projectId) {
-    return (
-      <WarningPanel title="Techdocs plugin:">
-        <pre>{TECHDOCS_ANNOTATION}</pre> annotation is missing on the entity.
-        <br />
-        <a href="https://backstage.io/docs/features/techdocs/creating-and-publishing">
-          Getting Started
-        </a>
-      </WarningPanel>
-    );
+    return <MissingAnnotationEmptyState annotation={TECHDOCS_ANNOTATION} />;
   }
 
   return (

plugins/techdocs/src/api.test.ts

@@ -28,7 +28,7 @@ describe('TechDocsStorageApi', () => {
     const storageApi = new TechDocsStorageApi({ apiOrigin: DOC_STORAGE_URL });
 
     expect(storageApi.getBaseUrl('test.js', mockEntity, '')).toEqual(
-      `${DOC_STORAGE_URL}/docs/${mockEntity.kind}/${mockEntity.namespace}/${mockEntity.name}/test.js`,
+      `${DOC_STORAGE_URL}/docs/${mockEntity.namespace}/${mockEntity.kind}/${mockEntity.name}/test.js`,
     );
   });
 
@@ -36,7 +36,7 @@ describe('TechDocsStorageApi', () => {
     const storageApi = new TechDocsStorageApi({ apiOrigin: DOC_STORAGE_URL });
 
     expect(storageApi.getBaseUrl('test/', mockEntity, '')).toEqual(
-      `${DOC_STORAGE_URL}/docs/${mockEntity.kind}/${mockEntity.namespace}/${mockEntity.name}/test/`,
+      `${DOC_STORAGE_URL}/docs/${mockEntity.namespace}/${mockEntity.kind}/${mockEntity.name}/test/`,
     );
   });
 });

plugins/techdocs/src/api.ts

@@ -51,9 +51,7 @@ export class TechDocsApi implements TechDocs {
   async getMetadata(metadataType: string, entityId: ParsedEntityId) {
     const { kind, namespace, name } = entityId;
 
-    const requestUrl = `${this.apiOrigin}/metadata/${metadataType}/${kind}/${
-      namespace ? namespace : 'default'
-    }/${name}`;
+    const requestUrl = `${this.apiOrigin}/metadata/${metadataType}/${namespace}/${kind}/${name}`;
 
     const request = await fetch(`${requestUrl}`);
     const res = await request.json();
@@ -72,9 +70,7 @@ export class TechDocsStorageApi implements TechDocsStorage {
   async getEntityDocs(entityId: ParsedEntityId, path: string) {
     const { kind, namespace, name } = entityId;
 
-    const url = `${this.apiOrigin}/docs/${kind}/${
-      namespace ? namespace : 'default'
-    }/${name}/${path}`;
+    const url = `${this.apiOrigin}/docs/${namespace}/${kind}/${name}/${path}`;
 
     const request = await fetch(
       `${url.endsWith('/') ? url : `${url}/`}index.html`,
@@ -96,9 +92,7 @@ export class TechDocsStorageApi implements TechDocsStorage {
 
     return new URL(
       oldBaseUrl,
-      `${this.apiOrigin}/docs/${kind}/${
-        namespace ? namespace : 'default'
-      }/${name}/${path}`,
+      `${this.apiOrigin}/docs/${namespace}/${kind}/${name}/${path}`,
     ).toString();
   }
 }

plugins/techdocs/src/plugin.ts

@@ -48,7 +48,7 @@ export const rootRouteRef = createRouteRef({
 });
 
 export const rootDocsRouteRef = createRouteRef({
-  path: ':entityId/*',
+  path: ':namespace/:kind/:name/*',
   title: 'Docs',
 });
 

plugins/techdocs/src/reader/components/TechDocsHome.tsx

@@ -84,9 +84,9 @@ export const TechDocsHome = () => {
                     onClick={() =>
                       navigate(
                         generatePath(rootDocsRouteRef.path, {
-                          entityId: `${entity.kind}:${
-                            entity.metadata.namespace ?? ''
-                          }:${entity.metadata.name}`,
+                          namespace: entity.metadata.namespace ?? 'default',
+                          kind: entity.kind,
+                          name: entity.metadata.name,
                         }),
                       )
                     }

plugins/techdocs/src/reader/components/TechDocsPage.tsx

@@ -24,8 +24,7 @@ import { techdocsApiRef } from '../../api';
 
 export const TechDocsPage = () => {
   const [documentReady, setDocumentReady] = useState<boolean>(false);
-  const { entityId } = useParams();
-  const [kind, namespace, name] = entityId.split(':');
+  const { namespace, kind, name } = useParams();
 
   const techDocsApi = useApi(techdocsApiRef);