Merge pull request #7568 from backstage/firestore-keystore

auth-backend: Add Firestore as new key-store provider

plugins/auth-backend/config.d.ts

@@ -31,6 +31,32 @@ export interface Config {
       secret?: string;
     };
 
+    /** To control how to store JWK data in auth-backend */
+    keyStore?: {
+      provider?: 'database' | 'memory' | 'firestore';
+      firestore?: {
+        /** The host to connect to */
+        host?: string;
+        /** The port to connect to */
+        port?: number;
+        /** Whether to use SSL when connecting. */
+        ssl?: boolean;
+        /** The Google Cloud Project ID */
+        projectId?: string;
+        /**
+         * Local file containing the Service Account credentials.
+         * You can omit this value to automatically read from
+         * GOOGLE_APPLICATION_CREDENTIALS env which is useful for local
+         * development.
+         */
+        keyFilename?: string;
+        /** The path to use for the collection. Defaults to 'sessions' */
+        path?: string;
+        /** Timeout used for database operations. Defaults to 10000ms */
+        timeout?: number;
+      };
+    };
+
     /**
      * The available auth-provider options and attributes
      */

plugins/auth-backend/package.json

@@ -36,6 +36,7 @@
     "@backstage/config": "^0.1.10",
     "@backstage/errors": "^0.1.1",
     "@backstage/test-utils": "^0.1.19",
+    "@google-cloud/firestore": "^4.15.1",
     "@types/express": "^4.17.6",
     "@types/passport": "^1.0.3",
     "compression": "^1.7.4",

plugins/auth-backend/src/identity/FirestoreKeyStore.test.ts

@@ -0,0 +1,184 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { Firestore } from '@google-cloud/firestore';
+
+import {
+  DEFAULT_DOCUMENT_PATH,
+  DEFAULT_TIMEOUT_MS,
+  FirestoreKeyStore,
+  FirestoreKeyStoreSettings,
+} from './FirestoreKeyStore';
+import { AnyJWK } from './types';
+
+const data = jest.fn().mockReturnValue('data');
+const toDate = jest.fn().mockReturnValue('date');
+const get = jest.fn().mockReturnValue({
+  docs: [{ data, createTime: { toDate } }],
+});
+const set = jest.fn();
+
+const firestoreMock = {
+  limit: jest.fn().mockReturnThis(),
+  collection: jest.fn().mockReturnThis(),
+  delete: jest.fn(),
+  doc: jest.fn().mockReturnThis(),
+  set,
+  get,
+};
+
+jest.mock('@google-cloud/firestore', () => ({
+  Firestore: jest.fn().mockImplementation(() => firestoreMock),
+}));
+
+describe('FirestoreKeyStore', () => {
+  const key = {
+    kid: '123',
+    use: 'sig',
+    kty: 'plain',
+    alg: 'Base64',
+  } as AnyJWK;
+
+  const settings = {
+    projectId: 'my-project',
+    host: 'my-host',
+    port: 8080,
+    ssl: false,
+    keyFilename: 'cred.json',
+  };
+  const path = 'my-path';
+  const timeout = 10;
+  const firestoreSettings = {
+    ...settings,
+    path,
+    timeout,
+  } as FirestoreKeyStoreSettings;
+
+  beforeEach(() => {
+    jest.clearAllMocks();
+    jest.useFakeTimers();
+  });
+
+  afterEach(() => {
+    jest.useRealTimers();
+  });
+
+  it('can create an instance without settings', async () => {
+    const keyStore = await FirestoreKeyStore.create();
+
+    expect(keyStore).toBeInstanceOf(FirestoreKeyStore);
+    expect(Firestore).toHaveBeenCalledWith({});
+  });
+
+  it('can create an instance with settings', async () => {
+    await FirestoreKeyStore.create(firestoreSettings);
+
+    expect(Firestore).toHaveBeenCalledWith(settings);
+  });
+
+  it('can verify that is has a connection to the database', async () => {
+    const keyStore = await FirestoreKeyStore.create();
+
+    await expect(
+      FirestoreKeyStore.verifyConnection(keyStore),
+    ).resolves.not.toThrow();
+  });
+
+  it('can verify that it can not connect to the database', async () => {
+    const keyStore = await FirestoreKeyStore.create();
+    firestoreMock.get = jest.fn().mockRejectedValue(new Error());
+
+    await expect(
+      FirestoreKeyStore.verifyConnection(keyStore),
+    ).rejects.toThrow();
+
+    firestoreMock.get = get;
+  });
+
+  it('can use a default timeout and path', async () => {
+    const keyStore = await FirestoreKeyStore.create();
+    await keyStore.addKey(key);
+
+    expect(setTimeout).toBeCalledWith(expect.any(Function), DEFAULT_TIMEOUT_MS);
+    expect(firestoreMock.collection).toBeCalledWith(DEFAULT_DOCUMENT_PATH);
+  });
+
+  it('can handle a timeout', async () => {
+    firestoreMock.set = jest
+      .fn()
+      .mockImplementation(
+        () => new Promise(resolve => setTimeout(resolve, 20)),
+      );
+    const keyStore = await FirestoreKeyStore.create(firestoreSettings);
+    const add = keyStore.addKey(key);
+
+    jest.advanceTimersByTime(50);
+
+    await expect(add).rejects.toEqual(
+      new Error(`Operation timed out after ${timeout}ms`),
+    );
+
+    firestoreMock.set = set;
+  });
+
+  it('can add keys', async () => {
+    const keyStore = await FirestoreKeyStore.create(firestoreSettings);
+    await keyStore.addKey(key);
+
+    expect(setTimeout).toBeCalledTimes(1);
+    expect(firestoreMock.collection).toBeCalledWith(path);
+    expect(firestoreMock.doc).toBeCalledWith(key.kid);
+    expect(firestoreMock.set).toHaveBeenCalledWith({
+      kid: key.kid,
+      key: JSON.stringify(key),
+    });
+  });
+
+  it('can delete a single key', async () => {
+    const keyStore = await FirestoreKeyStore.create(firestoreSettings);
+    await keyStore.removeKeys(['123']);
+
+    expect(setTimeout).toBeCalledTimes(1);
+    expect(firestoreMock.collection).toBeCalledWith(path);
+    expect(firestoreMock.doc).toBeCalledWith('123');
+    expect(firestoreMock.delete).toBeCalledTimes(1);
+  });
+
+  it('can delete a multiple keys', async () => {
+    const keyStore = await FirestoreKeyStore.create(firestoreSettings);
+    await keyStore.removeKeys(['123', '456']);
+
+    expect(setTimeout).toBeCalledTimes(2);
+    expect(firestoreMock.collection).toBeCalledWith(path);
+    expect(firestoreMock.doc).toBeCalledWith('123');
+    expect(firestoreMock.doc).toBeCalledWith('456');
+    expect(firestoreMock.delete).toBeCalledTimes(2);
+  });
+
+  it('can list keys', async () => {
+    const keyStore = await FirestoreKeyStore.create(firestoreSettings);
+    const items = await keyStore.listKeys();
+
+    expect(setTimeout).toBeCalledTimes(1);
+    expect(firestoreMock.collection).toBeCalledWith(path);
+    expect(firestoreMock.get).toBeCalledTimes(1);
+    expect(data).toBeCalledTimes(1);
+    expect(toDate).toBeCalledTimes(1);
+    expect(items).toMatchObject({
+      items: [{ key: 'data', createdAt: 'date' }],
+    });
+  });
+});

plugins/auth-backend/src/identity/FirestoreKeyStore.ts

@@ -0,0 +1,157 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { Logger } from 'winston';
+import {
+  DocumentData,
+  Firestore,
+  QuerySnapshot,
+  Settings,
+  WriteResult,
+} from '@google-cloud/firestore';
+
+import { AnyJWK, KeyStore, StoredKey } from './types';
+
+export type FirestoreKeyStoreSettings = Settings & Options;
+
+type Options = {
+  path?: string;
+  timeout?: number;
+};
+
+export const DEFAULT_TIMEOUT_MS = 10000;
+export const DEFAULT_DOCUMENT_PATH = 'sessions';
+
+export class FirestoreKeyStore implements KeyStore {
+  static async create(
+    settings?: FirestoreKeyStoreSettings,
+  ): Promise<FirestoreKeyStore> {
+    const { path, timeout, ...firestoreSettings } = settings ?? {};
+    const database = new Firestore(firestoreSettings);
+
+    return new FirestoreKeyStore(
+      database,
+      path ?? DEFAULT_DOCUMENT_PATH,
+      timeout ?? DEFAULT_TIMEOUT_MS,
+    );
+  }
+
+  private constructor(
+    private readonly database: Firestore,
+    private readonly path: string,
+    private readonly timeout: number,
+  ) {}
+
+  static async verifyConnection(
+    keyStore: FirestoreKeyStore,
+    logger?: Logger,
+  ): Promise<void> {
+    try {
+      await keyStore.verify();
+    } catch (error) {
+      if (process.env.NODE_ENV !== 'development') {
+        throw new Error(
+          `Failed to connect to database: ${(error as Error).message}`,
+        );
+      }
+      logger?.warn(
+        `Failed to connect to database: ${(error as Error).message}`,
+      );
+    }
+  }
+
+  async addKey(key: AnyJWK): Promise<void> {
+    await this.withTimeout<WriteResult>(
+      this.database
+        .collection(this.path)
+        .doc(key.kid)
+        .set({
+          kid: key.kid,
+          key: JSON.stringify(key),
+        }),
+    );
+  }
+
+  async listKeys(): Promise<{ items: StoredKey[] }> {
+    const keys = await this.withTimeout<QuerySnapshot<DocumentData>>(
+      this.database.collection(this.path).get(),
+    );
+
+    return {
+      items: keys.docs.map(key => ({
+        key: key.data() as AnyJWK,
+        createdAt: key.createTime.toDate(),
+      })),
+    };
+  }
+
+  async removeKeys(kids: string[]): Promise<void> {
+    // This is probably really slow, but it's done async in the background
+    for (const kid of kids) {
+      await this.withTimeout<WriteResult>(
+        this.database.collection(this.path).doc(kid).delete(),
+      );
+    }
+
+    /**
+     * This could be achieved with batching but there's a couple of limitations with that:
+     *
+     * - A batched write can contain a maximum of 500 operations
+     *  https://firebase.google.com/docs/firestore/manage-data/transactions#batched-writes
+     *
+     * - The "in" operator can combine a maximum of 10 equality clauses
+     *  https://firebase.google.com/docs/firestore/query-data/queries#in_not-in_and_array-contains-any
+     *
+     * Example:
+     *
+     *  const batch = this.database.batch();
+     *  const docs = await this.database
+     *    .collection(this.path)
+     *    .where('kid', 'in', kids)
+     *    .get();
+     *  docs.forEach(doc => {
+     *    batch.delete(doc.ref);
+     *  });
+     *  await batch.commit();
+     *
+     */
+  }
+
+  /**
+   * Helper function to allow us to modify the timeout used when
+   * performing Firestore database operations.
+   *
+   * The reason for this is that it seems that there's no other
+   * practical solution to change the default timeout of 10mins
+   * that Firestore has.
+   *
+   */
+  private async withTimeout<T>(operation: Promise<T>): Promise<T> {
+    const timer = new Promise<never>((_, reject) =>
+      setTimeout(() => {
+        reject(new Error(`Operation timed out after ${this.timeout}ms`));
+      }, this.timeout),
+    );
+    return Promise.race<T>([operation, timer]);
+  }
+
+  /**
+   * Used to verify that the database is reachable.
+   */
+  private async verify(): Promise<void> {
+    await this.withTimeout(this.database.collection(this.path).limit(1).get());
+  }
+}

plugins/auth-backend/src/identity/KeyStores.test.ts

@@ -0,0 +1,98 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { DatabaseManager } from '@backstage/backend-common';
+import { ConfigReader } from '@backstage/config';
+
+import { MemoryKeyStore } from './MemoryKeyStore';
+import { DatabaseKeyStore } from './DatabaseKeyStore';
+import { FirestoreKeyStore } from './FirestoreKeyStore';
+import { KeyStores } from './KeyStores';
+
+describe('KeyStores', () => {
+  const defaultConfigOptions = {
+    auth: {
+      keyStore: {
+        provider: 'memory',
+      },
+    },
+  };
+  const defaultConfig = new ConfigReader(defaultConfigOptions);
+
+  it('reads auth section from config', async () => {
+    const configSpy = jest.spyOn(defaultConfig, 'getOptionalConfig');
+    const keyStore = await KeyStores.fromConfig(defaultConfig);
+
+    expect(keyStore).toBeInstanceOf(MemoryKeyStore);
+    expect(configSpy).toHaveBeenCalledWith('auth.keyStore');
+    expect(
+      defaultConfig
+        .getOptionalConfig('auth.keyStore')
+        ?.getOptionalString('provider'),
+    ).toBe(defaultConfigOptions.auth.keyStore.provider);
+  });
+
+  it('can handle without auth config', async () => {
+    const config = new ConfigReader({
+      backend: {
+        database: {
+          client: 'sqlite3',
+          connection: ':memory:',
+        },
+      },
+    });
+    const database =
+      DatabaseManager.fromConfig(config).forPlugin('auth-backend');
+    const keyStore = await KeyStores.fromConfig(config, { database });
+
+    expect(keyStore).toBeInstanceOf(DatabaseKeyStore);
+  });
+
+  it('can handle additional provider config', async () => {
+    jest.spyOn(FirestoreKeyStore, 'verifyConnection').mockResolvedValue();
+    const createSpy = jest.spyOn(FirestoreKeyStore, 'create');
+
+    const configOptions = {
+      auth: {
+        keyStore: {
+          provider: 'firestore',
+          firestore: {
+            projectId: 'my-project',
+            keyFilename: 'cred.json',
+            path: 'my-path',
+            timeout: 100,
+            host: 'localhost',
+            port: 8088,
+            ssl: false,
+          },
+        },
+      },
+    };
+    const config = new ConfigReader(configOptions);
+    const keyStore = await KeyStores.fromConfig(config);
+
+    expect(keyStore).toBeInstanceOf(FirestoreKeyStore);
+    expect(createSpy).toHaveBeenCalledWith(
+      configOptions.auth.keyStore.firestore,
+    );
+    expect(
+      config
+        .getOptionalConfig('auth.keyStore')
+        ?.getOptionalConfig('firestore')
+        ?.getOptionalString('projectId'),
+    ).toBe(configOptions.auth.keyStore.firestore.projectId);
+  });
+});

plugins/auth-backend/src/identity/KeyStores.ts

@@ -0,0 +1,84 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { Logger } from 'winston';
+
+import { PluginDatabaseManager } from '@backstage/backend-common';
+import { Config } from '@backstage/config';
+
+import { DatabaseKeyStore } from './DatabaseKeyStore';
+import { MemoryKeyStore } from './MemoryKeyStore';
+import { FirestoreKeyStore } from './FirestoreKeyStore';
+import { KeyStore } from './types';
+
+type Options = {
+  logger?: Logger;
+  database?: PluginDatabaseManager;
+};
+
+export class KeyStores {
+  /**
+   * Looks at the `auth.keyStore` section in the application configuration
+   * and returns a KeyStore store. Defaults to `database`
+   *
+   * @returns a KeyStore store
+   */
+  static async fromConfig(
+    config: Config,
+    options?: Options,
+  ): Promise<KeyStore> {
+    const { logger, database } = options ?? {};
+
+    const ks = config.getOptionalConfig('auth.keyStore');
+    const provider = ks?.getOptionalString('provider') ?? 'database';
+
+    logger?.info(`Configuring "${provider}" as KeyStore provider`);
+
+    if (provider === 'database') {
+      if (!database) {
+        throw new Error('This KeyStore provider requires a database');
+      }
+
+      return await DatabaseKeyStore.create({
+        database: await database.getClient(),
+      });
+    }
+
+    if (provider === 'memory') {
+      return new MemoryKeyStore();
+    }
+
+    if (provider === 'firestore') {
+      const settings = ks?.getConfig(provider);
+
+      const keyStore = await FirestoreKeyStore.create({
+        projectId: settings?.getOptionalString('projectId'),
+        keyFilename: settings?.getOptionalString('keyFilename'),
+        host: settings?.getOptionalString('host'),
+        port: settings?.getOptionalNumber('port'),
+        ssl: settings?.getOptionalBoolean('ssl'),
+        path: settings?.getOptionalString('path'),
+        timeout: settings?.getOptionalNumber('timeout'),
+      });
+
+      await FirestoreKeyStore.verifyConnection(keyStore, logger);
+
+      return keyStore;
+    }
+
+    throw new Error(`Unknown KeyStore provider: ${provider}`);
+  }
+}

plugins/auth-backend/src/identity/index.ts

@@ -18,4 +18,7 @@ export { createOidcRouter } from './router';
 export { IdentityClient } from './IdentityClient';
 export { TokenFactory } from './TokenFactory';
 export { DatabaseKeyStore } from './DatabaseKeyStore';
+export { MemoryKeyStore } from './MemoryKeyStore';
+export { FirestoreKeyStore } from './FirestoreKeyStore';
+export { KeyStores } from './KeyStores';
 export type { KeyStore, TokenIssuer, TokenParams } from './types';

plugins/auth-backend/src/service/router.ts

@@ -29,7 +29,7 @@ import {
 import { assertError, NotFoundError } from '@backstage/errors';
 import { CatalogClient } from '@backstage/catalog-client';
 import { Config } from '@backstage/config';
-import { createOidcRouter, DatabaseKeyStore, TokenFactory } from '../identity';
+import { createOidcRouter, TokenFactory, KeyStores } from '../identity';
 import session from 'express-session';
 import passport from 'passport';
 import { Minimatch } from 'minimatch';
@@ -56,11 +56,9 @@ export async function createRouter({
   const appUrl = config.getString('app.baseUrl');
   const authUrl = await discovery.getExternalBaseUrl('auth');
 
+  const keyStore = await KeyStores.fromConfig(config, { logger, database });
   const keyDurationSeconds = 3600;
 
-  const keyStore = await DatabaseKeyStore.create({
-    database: await database.getClient(),
-  });
   const tokenIssuer = new TokenFactory({
     issuer: authUrl,
     keyStore,