Merge pull request #10428 from spreadshirt/kubernetes-cluster-refresh

Make Kubernetes clusters refreshable
This commit is contained in:
Fredrik Adelöw
2022-04-11 14:21:15 +02:00
committed by GitHub
13 changed files with 347 additions and 94 deletions
+61
View File
@@ -0,0 +1,61 @@
---
'@backstage/plugin-kubernetes-backend': minor
---
**BREAKING** Custom cluster suppliers need to cache their getClusters result
To allow custom `KubernetesClustersSupplier` instances to refresh the list of clusters
the `getClusters` method is now called whenever the list of clusters is needed.
Existing `KubernetesClustersSupplier` implementations will need to ensure that `getClusters`
can be called frequently and should return a cached result from `getClusters` instead.
For example, here's a simple example of a custom supplier in `packages/backend/src/plugins/kubernetes.ts`:
```diff
-import { KubernetesBuilder } from '@backstage/plugin-kubernetes-backend';
+import {
+ ClusterDetails,
+ KubernetesBuilder,
+ KubernetesClustersSupplier,
+} from '@backstage/plugin-kubernetes-backend';
import { Router } from 'express';
import { PluginEnvironment } from '../types';
+import { Duration } from 'luxon';
+
+export class CustomClustersSupplier implements KubernetesClustersSupplier {
+ constructor(private clusterDetails: ClusterDetails[] = []) {}
+
+ static create(refreshInterval: Duration) {
+ const clusterSupplier = new CustomClustersSupplier();
+ // setup refresh, e.g. using a copy of https://github.com/backstage/backstage/blob/master/plugins/search-backend-node/src/runPeriodically.ts
+ runPeriodically(
+ () => clusterSupplier.refreshClusters(),
+ refreshInterval.toMillis(),
+ );
+ return clusterSupplier;
+ }
+
+ async refreshClusters(): Promise<void> {
+ this.clusterDetails = []; // fetch from somewhere
+ }
+
+ async getClusters(): Promise<ClusterDetails[]> {
+ return this.clusterDetails;
+ }
+}
export default async function createPlugin(
env: PluginEnvironment,
): Promise<Router> {
- const { router } = await KubernetesBuilder.createBuilder({
+ const builder = await KubernetesBuilder.createBuilder({
logger: env.logger,
config: env.config,
- }).build();
+ });
+ builder.setClusterSupplier(
+ CustomClustersSupplier.create(Duration.fromObject({ minutes: 60 })),
+ );
+ const { router } = await builder.build();
```
+10
View File
@@ -57,6 +57,10 @@ This is an array used to determine where to retrieve cluster configuration from.
Valid cluster locator methods are:
- [`config`](#config)
- [`gke`](#gke)
- [custom `KubernetesClustersSupplier`](#custom-kubernetesclusterssupplier)
#### `config`
This cluster locator method will read cluster information from your app-config
@@ -261,6 +265,12 @@ Kubernetes plugin.
Defaults to `false`.
#### Custom `KubernetesClustersSupplier`
If the configuration-based cluster locators do not work for your use-case,
it is also possible to implement a
[custom `KubernetesClustersSupplier`](installation.md#custom-cluster-discovery).
### `customResources` (optional)
Configures which [custom resources][3] to look for when returning an entity's
+57
View File
@@ -90,6 +90,63 @@ async function main() {
That's it! The Kubernetes frontend and backend have now been added to your
Backstage app.
### Custom cluster discovery
If either existing
[cluster locators](https://backstage.io/docs/features/kubernetes/configuration#clusterlocatormethods)
don't work for your use-case, it is possible to implement a custom
[KubernetesClustersSupplier](https://backstage.io/docs/reference/plugin-kubernetes-backend.kubernetesclusterssupplier).
Change the following in `packages/backend/src/plugin/kubernetes.ts`:
```diff
-import { KubernetesBuilder } from '@backstage/plugin-kubernetes-backend';
+import {
+ ClusterDetails,
+ KubernetesBuilder,
+ KubernetesClustersSupplier,
+} from '@backstage/plugin-kubernetes-backend';
import { Router } from 'express';
import { PluginEnvironment } from '../types';
+import { Duration } from 'luxon';
+
+export class CustomClustersSupplier implements KubernetesClustersSupplier {
+ constructor(private clusterDetails: ClusterDetails[] = []) {}
+
+ static create(refreshInterval: Duration) {
+ const clusterSupplier = new CustomClustersSupplier();
+ // setup refresh, e.g. using a copy of https://github.com/backstage/backstage/blob/master/plugins/search-backend-node/src/runPeriodically.ts
+ runPeriodically(
+ () => clusterSupplier.refreshClusters(),
+ refreshInterval.toMillis(),
+ );
+ return clusterSupplier;
+ }
+
+ async refreshClusters(): Promise<void> {
+ this.clusterDetails = []; // fetch from somewhere
+ }
+
+ async getClusters(): Promise<ClusterDetails[]> {
+ return this.clusterDetails;
+ }
+}
export default async function createPlugin(
env: PluginEnvironment,
): Promise<Router> {
- const { router } = await KubernetesBuilder.createBuilder({
+ const builder = await KubernetesBuilder.createBuilder({
logger: env.logger,
config: env.config,
- }).build();
+ });
+ builder.setClusterSupplier(
+ CustomClustersSupplier.create(Duration.fromObject({ minutes: 60 })),
+ );
+ const { router } = await builder.build();
```
## Running Backstage locally
Start the frontend and the backend app by
+10 -7
View File
@@ -4,6 +4,7 @@
```ts
import { Config } from '@backstage/config';
import { Duration } from 'luxon';
import express from 'express';
import type { FetchResponse } from '@backstage/plugin-kubernetes-common';
import type { JsonObject } from '@backstage/types';
@@ -85,18 +86,20 @@ export class KubernetesBuilder {
// (undocumented)
build(): KubernetesBuilderReturn;
// (undocumented)
protected buildClusterSupplier(): KubernetesClustersSupplier;
protected buildClusterSupplier(
refreshInterval: Duration,
): KubernetesClustersSupplier;
// (undocumented)
protected buildCustomResources(): CustomResource[];
// (undocumented)
protected buildFetcher(): KubernetesFetcher;
// (undocumented)
protected buildHttpServiceLocator(
_clusterDetails: ClusterDetails[],
_clusterSupplier: KubernetesClustersSupplier,
): KubernetesServiceLocator;
// (undocumented)
protected buildMultiTenantServiceLocator(
clusterDetails: ClusterDetails[],
clusterSupplier: KubernetesClustersSupplier,
): KubernetesServiceLocator;
// (undocumented)
protected buildObjectsProvider(
@@ -105,12 +108,12 @@ export class KubernetesBuilder {
// (undocumented)
protected buildRouter(
objectsProvider: KubernetesObjectsProvider,
clusterDetails: ClusterDetails[],
clusterSupplier: KubernetesClustersSupplier,
): express.Router;
// (undocumented)
protected buildServiceLocator(
method: ServiceLocatorMethod,
clusterDetails: ClusterDetails[],
clusterSupplier: KubernetesClustersSupplier,
): KubernetesServiceLocator;
// (undocumented)
static createBuilder(env: KubernetesEnvironment): KubernetesBuilder;
@@ -127,6 +130,8 @@ export class KubernetesBuilder {
// (undocumented)
setClusterSupplier(clusterSupplier?: KubernetesClustersSupplier): this;
// (undocumented)
setDefaultClusterRefreshInterval(refreshInterval: Duration): this;
// (undocumented)
setFetcher(fetcher?: KubernetesFetcher): this;
// (undocumented)
setObjectsProvider(objectsProvider?: KubernetesObjectsProvider): this;
@@ -137,7 +142,6 @@ export class KubernetesBuilder {
// @public
export type KubernetesBuilderReturn = Promise<{
router: express.Router;
clusterDetails: ClusterDetails[];
clusterSupplier: KubernetesClustersSupplier;
customResources: CustomResource[];
fetcher: KubernetesFetcher;
@@ -149,7 +153,6 @@ export type KubernetesBuilderReturn = Promise<{
//
// @public (undocumented)
export interface KubernetesClustersSupplier {
// (undocumented)
getClusters(): Promise<ClusterDetails[]>;
}
+4 -2
View File
@@ -43,6 +43,7 @@
"@google-cloud/container": "^3.0.0",
"@kubernetes/client-node": "^0.16.0",
"@types/express": "^4.17.6",
"@types/luxon": "^2.0.4",
"aws-sdk": "^2.840.0",
"aws4": "^1.11.0",
"compression": "^1.7.4",
@@ -52,6 +53,7 @@
"fs-extra": "10.0.1",
"helmet": "^5.0.2",
"lodash": "^4.17.21",
"luxon": "^2.0.2",
"morgan": "^1.10.0",
"stream-buffers": "^3.0.2",
"winston": "^3.2.1",
@@ -60,8 +62,8 @@
"devDependencies": {
"@backstage/cli": "^0.17.0-next.1",
"@types/aws4": "^1.5.1",
"supertest": "^6.1.3",
"aws-sdk-mock": "^5.2.1"
"aws-sdk-mock": "^5.2.1",
"supertest": "^6.1.3"
},
"files": [
"dist",
@@ -17,7 +17,13 @@
import { Config } from '@backstage/config';
import { ForwardedError } from '@backstage/errors';
import * as container from '@google-cloud/container';
import { GKEClusterDetails, KubernetesClustersSupplier } from '../types/types';
import { Duration } from 'luxon';
import { runPeriodically } from '../service/runPeriodically';
import {
ClusterDetails,
GKEClusterDetails,
KubernetesClustersSupplier,
} from '../types/types';
type GkeClusterLocatorOptions = {
projectId: string;
@@ -31,11 +37,14 @@ export class GkeClusterLocator implements KubernetesClustersSupplier {
constructor(
private readonly options: GkeClusterLocatorOptions,
private readonly client: container.v1.ClusterManagerClient,
private clusterDetails: GKEClusterDetails[] | undefined = undefined,
private hasClusterDetails: boolean = false,
) {}
static fromConfigWithClient(
config: Config,
client: container.v1.ClusterManagerClient,
refreshInterval: Duration | undefined = undefined,
): GkeClusterLocator {
const options = {
projectId: config.getString('projectId'),
@@ -45,18 +54,37 @@ export class GkeClusterLocator implements KubernetesClustersSupplier {
config.getOptionalBoolean('skipMetricsLookup') ?? false,
exposeDashboard: config.getOptionalBoolean('exposeDashboard') ?? false,
};
return new GkeClusterLocator(options, client);
const gkeClusterLocator = new GkeClusterLocator(options, client);
if (refreshInterval) {
runPeriodically(
() => gkeClusterLocator.refreshClusters(),
refreshInterval.toMillis(),
);
}
return gkeClusterLocator;
}
static fromConfig(config: Config): GkeClusterLocator {
static fromConfig(
config: Config,
refreshInterval: Duration | undefined = undefined,
): GkeClusterLocator {
return GkeClusterLocator.fromConfigWithClient(
config,
new container.v1.ClusterManagerClient(),
refreshInterval,
);
}
async getClusters(): Promise<ClusterDetails[]> {
if (!this.hasClusterDetails) {
// refresh at least once when first called, when retries are disabled and in tests
await this.refreshClusters();
}
return this.clusterDetails ?? [];
}
// TODO pass caData into the object
async getClusters(): Promise<GKEClusterDetails[]> {
async refreshClusters(): Promise<void> {
const {
projectId,
region,
@@ -70,7 +98,7 @@ export class GkeClusterLocator implements KubernetesClustersSupplier {
try {
const [response] = await this.client.listClusters(request);
return (response.clusters ?? []).map(r => ({
this.clusterDetails = (response.clusters ?? []).map(r => ({
// TODO filter out clusters which don't have name or endpoint
name: r.name ?? 'unknown',
url: `https://${r.endpoint ?? ''}`,
@@ -88,6 +116,7 @@ export class GkeClusterLocator implements KubernetesClustersSupplier {
}
: {}),
}));
this.hasClusterDetails = true;
} catch (e) {
throw new ForwardedError(
`There was an error retrieving clusters from GKE for projectId=${projectId} region=${region}`,
@@ -15,9 +15,9 @@
*/
import { Config, ConfigReader } from '@backstage/config';
import { getCombinedClusterDetails } from './index';
import { getCombinedClusterSupplier } from './index';
describe('getCombinedClusterDetails', () => {
describe('getCombinedClusterSupplier', () => {
it('should retrieve cluster details from config', async () => {
const config: Config = new ConfigReader(
{
@@ -45,7 +45,8 @@ describe('getCombinedClusterDetails', () => {
'ctx',
);
const result = await getCombinedClusterDetails(config);
const clusterSupplier = getCombinedClusterSupplier(config);
const result = await clusterSupplier.getClusters();
expect(result).toStrictEqual([
{
@@ -99,7 +100,7 @@ describe('getCombinedClusterDetails', () => {
'ctx',
);
await expect(getCombinedClusterDetails(config)).rejects.toStrictEqual(
expect(() => getCombinedClusterSupplier(config)).toThrowError(
new Error('Unsupported kubernetes.clusterLocatorMethods: "magic"'),
);
});
@@ -15,38 +15,49 @@
*/
import { Config } from '@backstage/config';
import { ClusterDetails } from '../types/types';
import { Duration } from 'luxon';
import { ClusterDetails, KubernetesClustersSupplier } from '../types/types';
import { ConfigClusterLocator } from './ConfigClusterLocator';
import { GkeClusterLocator } from './GkeClusterLocator';
export const getCombinedClusterDetails = async (
class CombinedClustersSupplier implements KubernetesClustersSupplier {
constructor(readonly clusterSuppliers: KubernetesClustersSupplier[]) {}
async getClusters(): Promise<ClusterDetails[]> {
return await Promise.all(
this.clusterSuppliers.map(supplier => supplier.getClusters()),
)
.then(res => {
return res.flat();
})
.catch(e => {
throw e;
});
}
}
export const getCombinedClusterSupplier = (
rootConfig: Config,
): Promise<ClusterDetails[]> => {
return Promise.all(
rootConfig
.getConfigArray('kubernetes.clusterLocatorMethods')
.map(clusterLocatorMethod => {
const type = clusterLocatorMethod.getString('type');
switch (type) {
case 'config':
return ConfigClusterLocator.fromConfig(
clusterLocatorMethod,
).getClusters();
case 'gke':
return GkeClusterLocator.fromConfig(
clusterLocatorMethod,
).getClusters();
default:
throw new Error(
`Unsupported kubernetes.clusterLocatorMethods: "${type}"`,
);
}
}),
)
.then(res => {
return res.flat();
})
.catch(e => {
throw e;
refreshInterval: Duration | undefined = undefined,
): KubernetesClustersSupplier => {
const clusterSuppliers = rootConfig
.getConfigArray('kubernetes.clusterLocatorMethods')
.map(clusterLocatorMethod => {
const type = clusterLocatorMethod.getString('type');
switch (type) {
case 'config':
return ConfigClusterLocator.fromConfig(clusterLocatorMethod);
case 'gke':
return GkeClusterLocator.fromConfig(
clusterLocatorMethod,
refreshInterval,
);
default:
throw new Error(
`Unsupported kubernetes.clusterLocatorMethods: "${type}"`,
);
}
});
return new CombinedClustersSupplier(clusterSuppliers);
};
@@ -19,7 +19,9 @@ import { MultiTenantServiceLocator } from './MultiTenantServiceLocator';
describe('MultiTenantConfigClusterLocator', () => {
it('empty clusters returns empty cluster details', async () => {
const sut = new MultiTenantServiceLocator([]);
const sut = new MultiTenantServiceLocator({
getClusters: async () => [],
});
const result = await sut.getClustersByServiceId('ignored');
@@ -27,14 +29,18 @@ describe('MultiTenantConfigClusterLocator', () => {
});
it('one clusters returns one cluster details', async () => {
const sut = new MultiTenantServiceLocator([
{
name: 'cluster1',
url: 'http://localhost:8080',
authProvider: 'serviceAccount',
serviceAccountToken: '12345',
const sut = new MultiTenantServiceLocator({
getClusters: async () => {
return [
{
name: 'cluster1',
url: 'http://localhost:8080',
authProvider: 'serviceAccount',
serviceAccountToken: '12345',
},
];
},
]);
});
const result = await sut.getClustersByServiceId('ignored');
@@ -49,19 +55,23 @@ describe('MultiTenantConfigClusterLocator', () => {
});
it('two clusters returns two cluster details', async () => {
const sut = new MultiTenantServiceLocator([
{
name: 'cluster1',
serviceAccountToken: 'token',
url: 'http://localhost:8080',
authProvider: 'serviceAccount',
const sut = new MultiTenantServiceLocator({
getClusters: async () => {
return [
{
name: 'cluster1',
serviceAccountToken: 'token',
url: 'http://localhost:8080',
authProvider: 'serviceAccount',
},
{
name: 'cluster2',
url: 'http://localhost:8081',
authProvider: 'google',
},
];
},
{
name: 'cluster2',
url: 'http://localhost:8081',
authProvider: 'google',
},
]);
});
const result = await sut.getClustersByServiceId('ignored');
@@ -14,20 +14,24 @@
* limitations under the License.
*/
import { ClusterDetails, KubernetesServiceLocator } from '../types/types';
import {
ClusterDetails,
KubernetesClustersSupplier,
KubernetesServiceLocator,
} from '../types/types';
// This locator assumes that every service is located on every cluster
// Therefore it will always return all clusters provided
export class MultiTenantServiceLocator implements KubernetesServiceLocator {
private readonly clusterDetails: ClusterDetails[];
private readonly clusterSupplier: KubernetesClustersSupplier;
constructor(clusterDetails: ClusterDetails[]) {
this.clusterDetails = clusterDetails;
constructor(clusterSupplier: KubernetesClustersSupplier) {
this.clusterSupplier = clusterSupplier;
}
// As this implementation always returns all clusters serviceId is ignored here
// eslint-disable-next-line @typescript-eslint/no-unused-vars
async getClustersByServiceId(_serviceId: string): Promise<ClusterDetails[]> {
return this.clusterDetails;
return this.clusterSupplier.getClusters();
}
}
@@ -17,10 +17,10 @@ import { Config } from '@backstage/config';
import express from 'express';
import Router from 'express-promise-router';
import { Logger } from 'winston';
import { getCombinedClusterDetails } from '../cluster-locator';
import { Duration } from 'luxon';
import { getCombinedClusterSupplier } from '../cluster-locator';
import { MultiTenantServiceLocator } from '../service-locator/MultiTenantServiceLocator';
import {
ClusterDetails,
KubernetesObjectTypes,
ServiceLocatorMethod,
CustomResource,
@@ -50,7 +50,6 @@ export interface KubernetesEnvironment {
*/
export type KubernetesBuilderReturn = Promise<{
router: express.Router;
clusterDetails: ClusterDetails[];
clusterSupplier: KubernetesClustersSupplier;
customResources: CustomResource[];
fetcher: KubernetesFetcher;
@@ -60,6 +59,9 @@ export type KubernetesBuilderReturn = Promise<{
export class KubernetesBuilder {
private clusterSupplier?: KubernetesClustersSupplier;
private defaultClusterRefreshInterval: Duration = Duration.fromObject({
minutes: 60,
});
private objectsProvider?: KubernetesObjectsProvider;
private fetcher?: KubernetesFetcher;
private serviceLocator?: KubernetesServiceLocator;
@@ -91,13 +93,13 @@ export class KubernetesBuilder {
const fetcher = this.fetcher ?? this.buildFetcher();
const clusterSupplier = this.clusterSupplier ?? this.buildClusterSupplier();
const clusterDetails = await this.fetchClusterDetails(clusterSupplier);
const clusterSupplier =
this.clusterSupplier ??
this.buildClusterSupplier(this.defaultClusterRefreshInterval);
const serviceLocator =
this.serviceLocator ??
this.buildServiceLocator(this.getServiceLocatorMethod(), clusterDetails);
this.buildServiceLocator(this.getServiceLocatorMethod(), clusterSupplier);
const objectsProvider =
this.objectsProvider ??
@@ -109,10 +111,9 @@ export class KubernetesBuilder {
objectTypesToFetch: this.getObjectTypesToFetch(),
});
const router = this.buildRouter(objectsProvider, clusterDetails);
const router = this.buildRouter(objectsProvider, clusterSupplier);
return {
clusterDetails,
clusterSupplier,
customResources,
fetcher,
@@ -127,6 +128,11 @@ export class KubernetesBuilder {
return this;
}
public setDefaultClusterRefreshInterval(refreshInterval: Duration) {
this.defaultClusterRefreshInterval = refreshInterval;
return this;
}
public setObjectsProvider(objectsProvider?: KubernetesObjectsProvider) {
this.objectsProvider = objectsProvider;
return this;
@@ -161,13 +167,11 @@ export class KubernetesBuilder {
return customResources;
}
protected buildClusterSupplier(): KubernetesClustersSupplier {
protected buildClusterSupplier(
refreshInterval: Duration,
): KubernetesClustersSupplier {
const config = this.env.config;
return {
getClusters() {
return getCombinedClusterDetails(config);
},
};
return getCombinedClusterSupplier(config, refreshInterval);
}
protected buildObjectsProvider(
@@ -185,13 +189,13 @@ export class KubernetesBuilder {
protected buildServiceLocator(
method: ServiceLocatorMethod,
clusterDetails: ClusterDetails[],
clusterSupplier: KubernetesClustersSupplier,
): KubernetesServiceLocator {
switch (method) {
case 'multiTenant':
return this.buildMultiTenantServiceLocator(clusterDetails);
return this.buildMultiTenantServiceLocator(clusterSupplier);
case 'http':
return this.buildHttpServiceLocator(clusterDetails);
return this.buildHttpServiceLocator(clusterSupplier);
default:
throw new Error(
`Unsupported kubernetes.clusterLocatorMethod "${method}"`,
@@ -200,20 +204,20 @@ export class KubernetesBuilder {
}
protected buildMultiTenantServiceLocator(
clusterDetails: ClusterDetails[],
clusterSupplier: KubernetesClustersSupplier,
): KubernetesServiceLocator {
return new MultiTenantServiceLocator(clusterDetails);
return new MultiTenantServiceLocator(clusterSupplier);
}
protected buildHttpServiceLocator(
_clusterDetails: ClusterDetails[],
_clusterSupplier: KubernetesClustersSupplier,
): KubernetesServiceLocator {
throw new Error('not implemented');
}
protected buildRouter(
objectsProvider: KubernetesObjectsProvider,
clusterDetails: ClusterDetails[],
clusterSupplier: KubernetesClustersSupplier,
): express.Router {
const logger = this.env.logger;
const router = Router();
@@ -236,6 +240,7 @@ export class KubernetesBuilder {
});
router.get('/clusters', async (_, res) => {
const clusterDetails = await this.fetchClusterDetails(clusterSupplier);
res.json({
items: clusterDetails.map(cd => ({
name: cd.name,
@@ -0,0 +1,54 @@
/*
* Copyright 2020 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* Runs a function repeatedly, with a fixed wait between invocations.
*
* Supports async functions, and silently ignores exceptions and rejections.
*
* @param fn - The function to run. May return a Promise.
* @param delayMs - The delay between a completed function invocation and the
* next.
* @returns A function that, when called, stops the invocation loop.
*/
export function runPeriodically(fn: () => any, delayMs: number): () => void {
let cancel: () => void;
let cancelled = false;
const cancellationPromise = new Promise<void>(resolve => {
cancel = () => {
resolve();
cancelled = true;
};
});
const startRefresh = async () => {
while (!cancelled) {
try {
await fn();
} catch {
// ignore intentionally
}
await Promise.race([
new Promise(resolve => setTimeout(resolve, delayMs)),
cancellationPromise,
]);
}
};
startRefresh();
return cancel!;
}
@@ -80,6 +80,12 @@ export type KubernetesObjectTypes =
// Used to load cluster details from different sources
export interface KubernetesClustersSupplier {
/**
* Returns the cached list of clusters.
*
* Implementations _should_ cache the clusters and refresh them periodically,
* as getClusters is called whenever the list of clusters is needed.
*/
getClusters(): Promise<ClusterDetails[]>;
}