Merge pull request #19265 from ikatlinsky/feat/vault-allow-override-secret-engine

feature: Add ability to override default secretEngine value of vault plugin on catalog entity level
2023-09-12 12:18:50 +02:00
parent 438b7f46ff befd1149c5
commit 6d109af4e8
13 changed files with 230 additions and 62 deletions
@@ -0,0 +1,6 @@
+---
+'@backstage/plugin-vault-backend': patch
+'@backstage/plugin-vault': patch
+---
+
+Added ability to override vault secret engine value on catalog entity level using annotation `vault.io/secrets-engine`
@@ -66,7 +66,7 @@ To get started, first you need a running instance of Vault. You can follow [this
     baseUrl: http://your-internal-vault-url.svc
     publicUrl: https://your-vault-url.example.com
     token: <VAULT_TOKEN>
-     secretEngine: 'customSecretEngine' # Optional. By default it uses 'secrets'
+     secretEngine: 'customSecretEngine' # Optional. By default it uses 'secrets'. Can be overwritten by the annotation of the entity
     kvVersion: <kv-version> # Optional. The K/V version that your instance is using. The available options are '1' or '2'
   ```

@@ -106,6 +106,20 @@ The path is relative to your secrets engine folder. So if you want to get the se

 You will set the `vault.io/secret-path` to `test/backstage`. If the folder `backstage` contains other sub-folders, the plugin will fetch the secrets inside them and adapt the **View** and **Edit** URLs to point to the correct place.

+In case you need to support different secret engines for entities of the catalog you can provide optional annotation to the entity in `catalog-info.yaml`:
+
+```diff
+ apiVersion: backstage.io/v1alpha1
+ kind: Component
+ metadata:
+   # ...
+   annotations:
+     vault.io/secrets-path: path/to/secrets
+    vault.io/secrets-engine: customSecretEngine # Optional. By default it uses the 'secretEngine' value from your app-config.
+```
+
+That will overwrite the default secret engine from the configuration.
+
 ## Renew token

 In a secure Vault instance, it's usual that the tokens are refreshed after some time. In order to always have a valid token to fetch the secrets, it might be necessary to execute a renew action after some time. By default this is deactivated, but it can be easily activated and configured to be executed periodically (hourly by default, but customizable by the user). In order to do that, modify your `src/plugins/vault.ts` file to look like this one:
@@ -138,6 +152,7 @@ export default async function createPlugin(
 ## Features

 - List the secrets present in a certain path
+- Use different secret engines for different entities
 - Open a link to view the secret
 - Open a link to edit the secret
 - Renew the token automatically with a defined periodicity
@@ -25,7 +25,12 @@ export interface RouterOptions {
 // @public
 export interface VaultApi {
  getFrontendSecretsUrl(): string;
-  listSecrets(secretPath: string): Promise<VaultSecret[]>;
+  listSecrets(
+    secretPath: string,
+    options?: {
+      secretEngine?: string;
+    },
+  ): Promise<VaultSecret[]>;
  renewToken?(): Promise<void>;
 }

@@ -49,7 +54,12 @@ export class VaultClient implements VaultApi {
  // (undocumented)
  getFrontendSecretsUrl(): string;
  // (undocumented)
-  listSecrets(secretPath: string): Promise<VaultSecret[]>;
+  listSecrets(
+    secretPath: string,
+    options?: {
+      secretEngine?: string;
+    },
+  ): Promise<VaultSecret[]>;
  // (undocumented)
  renewToken(): Promise<void>;
 }
@@ -140,11 +140,18 @@ export class VaultBuilder {

    router.get('/v1/secrets/:path', async (req, res) => {
      const { path } = req.params;
+      const { engine } = req.query;
+
      if (typeof path !== 'string') {
        throw new InputError(`Invalid path: ${path}`);
      }
+      if (engine && typeof engine !== 'string') {
+        throw new InputError(`Invalid engine: ${engine}`);
+      }

-      const secrets = await vaultApi.listSecrets(path);
+      const secrets = await vaultApi.listSecrets(path, {
+        secretEngine: engine,
+      });
      res.json({ items: secrets });
    });

@@ -21,6 +21,8 @@ import { VaultSecret, VaultClient, VaultSecretList } from './vaultApi';
 import { ConfigReader } from '@backstage/config';

 describe('VaultApi', () => {
+  let api: VaultClient;
+
  const server = setupServer();
  setupRequestMockHandlers(server);

@@ -43,20 +45,24 @@ describe('VaultApi', () => {
    },
  };

-  const mockSecretsResult: VaultSecret[] = [
-    {
-      name: 'secret::one',
-      path: 'test/success',
-      editUrl: `${mockBaseUrl}/ui/vault/secrets/secrets/edit/test/success/secret::one`,
-      showUrl: `${mockBaseUrl}/ui/vault/secrets/secrets/show/test/success/secret::one`,
-    },
-    {
-      name: 'secret::two',
-      path: 'test/success',
-      editUrl: `${mockBaseUrl}/ui/vault/secrets/secrets/edit/test/success/secret::two`,
-      showUrl: `${mockBaseUrl}/ui/vault/secrets/secrets/show/test/success/secret::two`,
-    },
-  ];
+  const mockSecretsResult = (
+    secretEngine: string = 'secrets',
+  ): VaultSecret[] => {
+    return [
+      {
+        name: 'secret::one',
+        path: 'test/success',
+        editUrl: `${mockBaseUrl}/ui/vault/secrets/${secretEngine}/edit/test/success/secret::one`,
+        showUrl: `${mockBaseUrl}/ui/vault/secrets/${secretEngine}/show/test/success/secret::one`,
+      },
+      {
+        name: 'secret::two',
+        path: 'test/success',
+        editUrl: `${mockBaseUrl}/ui/vault/secrets/${secretEngine}/edit/test/success/secret::two`,
+        showUrl: `${mockBaseUrl}/ui/vault/secrets/${secretEngine}/show/test/success/secret::two`,
+      },
+    ];
+  };

  const setupHandlers = () => {
    server.use(
@@ -66,6 +72,9 @@ describe('VaultApi', () => {
          return res(ctx.json(mockListResult));
        },
      ),
+      rest.get(`${mockBaseUrl}/v1/kv/metadata/test/success`, (_, res, ctx) => {
+        return res(ctx.json(mockListResult));
+      }),
      rest.get(
        `${mockBaseUrl}/v1/secrets/metadata/test/error`,
        (_, res, ctx) => {
@@ -78,28 +87,33 @@ describe('VaultApi', () => {
    );
  };

-  it('should return secrets', async () => {
+  beforeEach(() => {
    setupHandlers();
-    const api = new VaultClient({ config });
+    api = new VaultClient({ config });
+  });
+
+  it('should return secrets', async () => {
    const secrets = await api.listSecrets('test/success');
-    expect(secrets).toEqual(mockSecretsResult);
+    expect(secrets).toEqual(mockSecretsResult());
+  });
+
+  it('should return secrets for custom engine', async () => {
+    const secrets = await api.listSecrets('test/success', {
+      secretEngine: 'kv',
+    });
+    expect(secrets).toEqual(mockSecretsResult('kv'));
  });

  it('should return empty secret list', async () => {
-    setupHandlers();
-    const api = new VaultClient({ config });
    const secrets = await api.listSecrets('test/error');
    expect(secrets).toEqual([]);
  });

  it('should return success token renew', async () => {
-    setupHandlers();
-    const api = new VaultClient({ config });
    expect(await api.renewToken()).toBe(undefined);
  });

  it('should render frontend url', () => {
-    const api = new VaultClient({ config });
    const url = api.getFrontendSecretsUrl();
    expect(url).toEqual(`${mockBaseUrl}/ui/vault/secrets/secrets`);
  });
@@ -59,11 +59,19 @@ export interface VaultApi {
   * Returns the URL to access the Vault UI with the defined config.
   */
  getFrontendSecretsUrl(): string;
+
  /**
   * Returns a list of secrets used to show in a table.
   * @param secretPath - The path where the secrets are stored in Vault
+   * @param options - Additional options to be passed to the Vault API, allows to override vault default settings in app config file
   */
-  listSecrets(secretPath: string): Promise<VaultSecret[]>;
+  listSecrets(
+    secretPath: string,
+    options?: {
+      secretEngine?: string;
+    },
+  ): Promise<VaultSecret[]>;
+
  /**
   * Optional, to renew the token used to list the secrets. Throws an
   * error if the token renewal went wrong.
@@ -115,11 +123,18 @@ export class VaultClient implements VaultApi {
    return `${this.vaultConfig.baseUrl}/ui/vault/secrets/${this.vaultConfig.secretEngine}`;
  }

-  async listSecrets(secretPath: string): Promise<VaultSecret[]> {
+  async listSecrets(
+    secretPath: string,
+    options?: {
+      secretEngine?: string;
+    },
+  ): Promise<VaultSecret[]> {
+    const mount = options?.secretEngine || this.vaultConfig.secretEngine;
+
    const listUrl =
      this.vaultConfig.kvVersion === 2
-        ? `v1/${this.vaultConfig.secretEngine}/metadata/${secretPath}`
-        : `v1/${this.vaultConfig.secretEngine}/${secretPath}`;
+        ? `v1/${encodeURIComponent(mount)}/metadata/${secretPath}`
+        : `v1/${encodeURIComponent(mount)}/${secretPath}`;
    const result = await this.limit(() =>
      this.callApi<VaultSecretList>(listUrl, { list: true }),
    );
@@ -131,7 +146,9 @@ export class VaultClient implements VaultApi {
        if (secret.endsWith('/')) {
          secrets.push(
            ...(await this.limit(() =>
-              this.listSecrets(`${secretPath}/${secret.slice(0, -1)}`),
+              this.listSecrets(`${secretPath}/${secret.slice(0, -1)}`, {
+                secretEngine: mount,
+              }),
            )),
          );
        } else {
@@ -140,8 +157,12 @@ export class VaultClient implements VaultApi {
          secrets.push({
            name: secret,
            path: secretPath,
-            editUrl: `${vaultUrl}/ui/vault/secrets/${this.vaultConfig.secretEngine}/edit/${secretPath}/${secret}`,
-            showUrl: `${vaultUrl}/ui/vault/secrets/${this.vaultConfig.secretEngine}/show/${secretPath}/${secret}`,
+            editUrl: `${vaultUrl}/ui/vault/secrets/${encodeURIComponent(
+              mount,
+            )}/edit/${secretPath}/${secret}`,
+            showUrl: `${vaultUrl}/ui/vault/secrets/${encodeURIComponent(
+              mount,
+            )}/show/${secretPath}/${secret}`,
          });
        }
      }),
@@ -42,7 +42,7 @@ To get started, first you need a running instance of Vault. You can follow [this
   vault:
     baseUrl: http://your-vault-url
     token: <VAULT_TOKEN>
-     secretEngine: 'customSecretEngine' # Optional. By default it uses 'secrets'
+     secretEngine: 'customSecretEngine' # Optional. By default it uses 'secrets'. Can be overwritten by the annotation of the entity
     kvVersion: <kv-version> # Optional. The K/V version that your instance is using. The available options are '1' or '2'
   ```

@@ -67,6 +67,7 @@ metadata:
  # ...
  annotations:
    vault.io/secrets-path: path/to/secrets
+    vault.io/secrets-engine: customSecretEngine # Optional. By default it uses the 'secretEngine' value from your app-config.
 ```

 The path is relative to your secrets engine folder. So if you want to get the secrets for backstage and you have the following directory structure:
@@ -86,9 +87,24 @@ If the annotation is missing for a certain component, then the card will show so

 ![Screenshot of the vault plugin with missing annotation](images/annotation-missing.png)

+In case you need to support different secret engines for entities of the catalog you can provide optional annotation to the entity in `catalog-info.yaml`:
+
+```diff
+ apiVersion: backstage.io/v1alpha1
+ kind: Component
+ metadata:
+   # ...
+   annotations:
+     vault.io/secrets-path: path/to/secrets
+    vault.io/secrets-engine: customSecretEngine # Optional. By default it uses 'secertEngine' value from configuration.
+```
+
+That will overwrite the default secret engine from the configuration.
+
 ## Features

 - List the secrets present in a certain path
+- Use different secret engines for different components
 - Open a link to view the secret
 - Open a link to edit the secret
 - Renew the token automatically with a defined periodicity
@@ -21,7 +21,12 @@ export const VAULT_SECRET_PATH_ANNOTATION = 'vault.io/secrets-path';

 // @public
 export interface VaultApi {
-  listSecrets(secretPath: string): Promise<VaultSecret[]>;
+  listSecrets(
+    secretPath: string,
+    options?: {
+      secretEngine?: string;
+    },
+  ): Promise<VaultSecret[]>;
 }

 // @public (undocumented)
@@ -21,6 +21,9 @@ import { VaultSecret, VaultClient } from './api';
 import { UrlPatternDiscovery } from '@backstage/core-app-api';

 describe('api', () => {
+  let api: VaultClient;
+  let fetchApiSpy: jest.SpyInstance<Promise<Response>>;
+
  const server = setupServer();
  setupRequestMockHandlers(server);

@@ -64,37 +67,50 @@ describe('api', () => {
    );
  };

-  it('should return secrets', async () => {
+  beforeEach(() => {
+    jest.resetAllMocks();
+
    setupHandlers();
-    const api = new VaultClient({ discoveryApi, fetchApi });
+
+    api = new VaultClient({ discoveryApi, fetchApi });
+    fetchApiSpy = jest.spyOn(fetchApi, 'fetch');
+  });
+
+  it('should return secrets', async () => {
    expect(await api.listSecrets('test/success')).toEqual(
      mockSecretsResult.items,
    );
+    expect(fetchApiSpy).toHaveBeenCalledWith(
+      `${mockBaseUrl}/v1/secrets/test%2Fsuccess?`,
+      expect.anything(),
+    );
+  });
+
+  it('should return secrets with custom engine', async () => {
+    expect(
+      await api.listSecrets('test/success', { secretEngine: 'kv' }),
+    ).toEqual(mockSecretsResult.items);
+    expect(fetchApiSpy).toHaveBeenCalledWith(
+      `${mockBaseUrl}/v1/secrets/test%2Fsuccess?engine=kv`,
+      expect.anything(),
+    );
  });

  it('should return empty secret list', async () => {
-    setupHandlers();
-    const api = new VaultClient({ discoveryApi, fetchApi });
    expect(await api.listSecrets('test/empty')).toEqual([]);
  });

  it('should return all the secrets if no path defined', async () => {
-    setupHandlers();
-    const api = new VaultClient({ discoveryApi, fetchApi });
    expect(await api.listSecrets('')).toEqual(mockSecretsResult.items);
  });

  it('should throw an error if the Vault API responds with an HTTP 404', async () => {
-    setupHandlers();
-    const api = new VaultClient({ discoveryApi, fetchApi });
    await expect(api.listSecrets('test/not-found')).rejects.toThrow(
      "No secrets found in path 'v1/secrets/test%2Fnot-found'",
    );
  });

  it('should throw an error if the Vault API responds with a non-successful HTTP status code', async () => {
-    setupHandlers();
-    const api = new VaultClient({ discoveryApi, fetchApi });
    await expect(api.listSecrets('test/error')).rejects.toThrow(
      'Request failed with 400 Error',
    );
@@ -46,8 +46,14 @@ export interface VaultApi {
  /**
   * Returns a list of secrets used to show in a table.
   * @param secretPath - The path where the secrets are stored in Vault
+   * @param options - Additional options to be passed to the Vault API, allows to override vault default settings in app config file
   */
-  listSecrets(secretPath: string): Promise<VaultSecret[]>;
+  listSecrets(
+    secretPath: string,
+    options?: {
+      secretEngine?: string;
+    },
+  ): Promise<VaultSecret[]>;
 }

 /**
@@ -90,10 +96,21 @@ export class VaultClient implements VaultApi {
    throw await ResponseError.fromResponse(response);
  }

-  async listSecrets(secretPath: string): Promise<VaultSecret[]> {
+  async listSecrets(
+    secretPath: string,
+    options?: {
+      secretEngine?: string;
+    },
+  ): Promise<VaultSecret[]> {
+    const query: { [key in string]: any } = {};
+    const { secretEngine } = options || {};
+    if (secretEngine) {
+      query.engine = secretEngine;
+    }
+
    const result = await this.callApi<{ items: VaultSecret[] }>(
      `v1/secrets/${encodeURIComponent(secretPath)}`,
-      {},
+      query,
    );
    return result.items;
  }
@@ -29,23 +29,31 @@ import { VaultSecret, vaultApiRef, VaultClient } from '../../api';
 import { rest } from 'msw';

 describe('EntityVaultTable', () => {
+  let vaultClient: VaultClient;
+  let apis: TestApiRegistry;
+  let listSecretsSpy: jest.SpyInstance<Promise<VaultSecret[]>>;
+
  const server = setupServer();
  setupRequestMockHandlers(server);
-  let apis: TestApiRegistry;
  const mockBaseUrl = 'https://api-vault.com/api/vault';
  const discoveryApi = UrlPatternDiscovery.compile(mockBaseUrl);
  const fetchApi = new MockFetchApi();

-  const entity = (secretPath: string) => {
+  const entity = (secretPath: string, secretEngine?: string | undefined) => {
+    const annotations: Record<string, string> = {
+      'vault.io/secrets-path': secretPath,
+    };
+    if (secretEngine) {
+      annotations['vault.io/secrets-engine'] = secretEngine;
+    }
+
    return {
      apiVersion: 'backstage.io/v1alpha1',
      kind: 'Component',
      metadata: {
        name: 'test',
        description: 'This is the description',
-        annotations: {
-          'vault.io/secrets-path': secretPath,
-        },
+        annotations,
      },
      spec: {
        lifecycle: 'production',
@@ -56,6 +64,7 @@ describe('EntityVaultTable', () => {
  };

  const entityOk = entity('test/success');
+  const entityOkWithEngine = entity('test/success', 'kv');
  const entityEmpty = entity('test/empty');
  const entityNotOk = entity('test/error');

@@ -91,10 +100,10 @@ describe('EntityVaultTable', () => {
  };

  beforeEach(() => {
-    apis = TestApiRegistry.from([
-      vaultApiRef,
-      new VaultClient({ discoveryApi, fetchApi }),
-    ]);
+    jest.resetAllMocks();
+    vaultClient = new VaultClient({ discoveryApi, fetchApi });
+    apis = TestApiRegistry.from([vaultApiRef, vaultClient]);
+    listSecretsSpy = jest.spyOn(vaultClient, 'listSecrets');
  });

  it('should render secrets', async () => {
@@ -107,6 +116,28 @@ describe('EntityVaultTable', () => {

    expect(await rendered.findAllByText(/secret::one/)).toBeDefined();
    expect(await rendered.findAllByText(/secret::two/)).toBeDefined();
+
+    expect(listSecretsSpy).toHaveBeenCalledTimes(1);
+    expect(listSecretsSpy).toHaveBeenCalledWith('test/success', {
+      secretEngine: undefined,
+    });
+  });
+
+  it('should render secrets with custom engine', async () => {
+    setupHandlers();
+    const rendered = await renderInTestApp(
+      <ApiProvider apis={apis}>
+        <EntityVaultTable entity={entityOkWithEngine} />
+      </ApiProvider>,
+    );
+
+    expect(await rendered.findAllByText(/secret::one/)).toBeDefined();
+    expect(await rendered.findAllByText(/secret::two/)).toBeDefined();
+
+    expect(listSecretsSpy).toHaveBeenCalledTimes(1);
+    expect(listSecretsSpy).toHaveBeenCalledWith('test/success', {
+      secretEngine: 'kv',
+    });
  });

  it('should render no secrets found', async () => {
@@ -23,27 +23,33 @@ import Visibility from '@material-ui/icons/Visibility';
 import Alert from '@material-ui/lab/Alert';
 import useAsync from 'react-use/lib/useAsync';
 import { VaultSecret, vaultApiRef } from '../../api';
-import { VAULT_SECRET_PATH_ANNOTATION } from '../../constants';
+import {
+  VAULT_SECRET_ENGINE_ANNOTATION,
+  VAULT_SECRET_PATH_ANNOTATION,
+} from '../../constants';

-export const vaultSecretPath = (entity: Entity) => {
+export const vaultSecretConfig = (entity: Entity) => {
  const secretPath =
    entity.metadata.annotations?.[VAULT_SECRET_PATH_ANNOTATION];
+  const secretEngine =
+    entity.metadata.annotations?.[VAULT_SECRET_ENGINE_ANNOTATION];

-  return { secretPath };
+  return { secretPath, secretEngine };
 };

 export const EntityVaultTable = ({ entity }: { entity: Entity }) => {
  const vaultApi = useApi(vaultApiRef);
-  const { secretPath } = vaultSecretPath(entity);
+  const { secretPath, secretEngine } = vaultSecretConfig(entity);
  if (!secretPath) {
    throw Error(
      `The secret path is undefined. Please, define the annotation ${VAULT_SECRET_PATH_ANNOTATION}`,
    );
  }
+
  const { value, loading, error } = useAsync(async (): Promise<
    VaultSecret[]
  > => {
-    return vaultApi.listSecrets(secretPath);
+    return vaultApi.listSecrets(secretPath, { secretEngine });
  }, []);

  const columns: TableColumn[] = [
@@ -14,6 +14,10 @@
 * limitations under the License.
 */

+/**
+ * @public
+ */
+export const VAULT_SECRET_ENGINE_ANNOTATION = 'vault.io/secrets-engine';
 /**
 * @public
 */