paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 26 Packages Projects Releases Wiki Activity

node-fetch-based implementation for pod metrics

Browse Source 
which eliminates the need for the KubernetesClientProvider

Signed-off-by: Jamie Klassen <jklassen@vmware.com>
This commit is contained in:
Jamie Klassen
2022-11-30 10:16:12 -05:00
parent f54362ad57
commit fcf8d33014
7 changed files with 129 additions and 318 deletions
Download Patch File Download Diff File Expand all files Collapse all files
plugins/kubernetes-backend/api-report.md
-16
View File
@@ -5,21 +5,17 @@
```ts
import { CatalogApi } from '@backstage/catalog-client';
import { Config } from '@backstage/config';
import { CoreV1Api } from '@kubernetes/client-node';
import { Credentials } from 'aws-sdk';
import { CustomObjectsApi } from '@kubernetes/client-node';
import type { CustomResourceMatcher } from '@backstage/plugin-kubernetes-common';
import { Duration } from 'luxon';
import { Entity } from '@backstage/catalog-model';
import express from 'express';
import type { FetchResponse } from '@backstage/plugin-kubernetes-common';
import type { JsonObject } from '@backstage/types';
import { KubeConfig } from '@kubernetes/client-node';
import type { KubernetesFetchError } from '@backstage/plugin-kubernetes-common';
import { KubernetesRequestAuth } from '@backstage/plugin-kubernetes-common';
import type { KubernetesRequestBody } from '@backstage/plugin-kubernetes-common';
import { Logger } from 'winston';
import { Metrics } from '@kubernetes/client-node';
import type { ObjectsByEntityResponse } from '@backstage/plugin-kubernetes-common';
import { PluginEndpointDiscovery } from '@backstage/backend-common';
import type { RequestHandler } from 'express';
@@ -262,18 +258,6 @@ export type KubernetesBuilderReturn = Promise<{
serviceLocator: KubernetesServiceLocator;
}>;
// @alpha (undocumented)
export class KubernetesClientProvider {
// (undocumented)
getCoreClientByClusterDetails(clusterDetails: ClusterDetails): CoreV1Api;
// (undocumented)
getCustomObjectsClient(clusterDetails: ClusterDetails): CustomObjectsApi;
// (undocumented)
getKubeConfig(clusterDetails: ClusterDetails): KubeConfig;
// (undocumented)
getMetricsClient(clusterDetails: ClusterDetails): Metrics;
}
// @alpha
export interface KubernetesClustersSupplier {
getClusters(): Promise<ClusterDetails[]>;
plugins/kubernetes-backend/src/service/KubernetesBuilder.ts
-2
View File
@@ -34,7 +34,6 @@ import {
ObjectsByEntityRequest,
ServiceLocatorMethod,
} from '../types/types';
import { KubernetesClientProvider } from './KubernetesClientProvider';
import {
DEFAULT_OBJECTS,
KubernetesFanOutHandler,
@@ -211,7 +210,6 @@ export class KubernetesBuilder {
protected buildFetcher(): KubernetesFetcher {
this.fetcher = new KubernetesClientBasedFetcher({
kubernetesClientProvider: new KubernetesClientProvider(),
logger: this.env.logger,
});
plugins/kubernetes-backend/src/service/KubernetesClientProvider.test.ts
-95
View File
@@ -1,95 +0,0 @@
/*
* Copyright 2020 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import '@backstage/backend-common';
import { KubernetesClientProvider } from './KubernetesClientProvider';
import { ClusterDetails } from '../types/types';
import * as https from 'https';
import mockFs from 'mock-fs';
describe('KubernetesClientProvider', () => {
beforeEach(() => {
jest.resetAllMocks();
});
afterEach(() => {
mockFs.restore();
});
it('can get core client by cluster details', () => {
const sut = new KubernetesClientProvider();
const getKubeConfig = jest.spyOn(sut, 'getKubeConfig');
const clusterDetails: ClusterDetails = {
name: 'cluster-name',
url: 'http://localhost:9999',
serviceAccountToken: 'TOKEN',
authProvider: 'serviceAccount',
};
const result = sut.getCoreClientByClusterDetails(clusterDetails);
expect(result.basePath).toBe('http://localhost:9999');
// These fields aren't on the type but are there
const auth = (result as any).authentications.default;
expect(auth.users[0].token).toBe('TOKEN');
expect(auth.clusters[0].name).toBe('cluster-name');
expect(auth.clusters[0].skipTLSVerify).toBe(false);
expect(getKubeConfig).toHaveBeenCalledTimes(1);
});
it('can get custom objects client by cluster details', () => {
const sut = new KubernetesClientProvider();
const getKubeConfig = jest.spyOn(sut, 'getKubeConfig');
const clusterDetails: ClusterDetails = {
name: 'cluster-name',
url: 'http://localhost:9999',
serviceAccountToken: 'TOKEN',
authProvider: 'serviceAccount',
skipTLSVerify: false,
};
const result = sut.getCustomObjectsClient(clusterDetails);
expect(result.basePath).toBe('http://localhost:9999');
// These fields aren't on the type but are there
const auth = (result as any).authentications.default;
expect(auth.users[0].token).toBe('TOKEN');
expect(auth.clusters[0].name).toBe('cluster-name');
expect(getKubeConfig).toHaveBeenCalledTimes(1);
});
it('respects caFile', async () => {
mockFs({
'/path/to/ca.crt': 'my-ca',
});
const clusterDetails: ClusterDetails = {
name: 'cluster-name',
url: 'https://localhost:9999',
authProvider: 'serviceAccount',
serviceAccountToken: 'TOKEN',
caFile: '/path/to/ca.crt',
};
const kubeConfig = new KubernetesClientProvider().getKubeConfig(
clusterDetails,
);
const options: https.RequestOptions = {};
await kubeConfig.applytoHTTPSOptions(options);
expect(options.ca?.toString()).toEqual('my-ca');
});
});
plugins/kubernetes-backend/src/service/KubernetesClientProvider.ts
-87
View File
@@ -1,87 +0,0 @@
/*
* Copyright 2020 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import {
Cluster,
Context,
CoreV1Api,
CustomObjectsApi,
KubeConfig,
Metrics,
User,
} from '@kubernetes/client-node';
import { ClusterDetails } from '../types/types';
/**
*
* @alpha
*/
export class KubernetesClientProvider {
// visible for testing
getKubeConfig(clusterDetails: ClusterDetails): KubeConfig {
const cluster: Cluster = {
name: clusterDetails.name,
server: clusterDetails.url,
skipTLSVerify: clusterDetails.skipTLSVerify || false,
caData: clusterDetails.caData,
caFile: clusterDetails.caFile,
};
// TODO configure
const user: User = {
name: 'backstage',
token: clusterDetails.serviceAccountToken,
};
const context: Context = {
name: `${clusterDetails.name}`,
user: user.name,
cluster: cluster.name,
};
const kc: KubeConfig = new KubeConfig();
if (clusterDetails.serviceAccountToken) {
kc.loadFromOptions({
clusters: [cluster],
users: [user],
contexts: [context],
currentContext: context.name,
});
} else {
kc.loadFromDefault();
}
return kc;
}
getCoreClientByClusterDetails(clusterDetails: ClusterDetails): CoreV1Api {
const kc = this.getKubeConfig(clusterDetails);
return kc.makeApiClient(CoreV1Api);
}
getMetricsClient(clusterDetails: ClusterDetails): Metrics {
const kc = this.getKubeConfig(clusterDetails);
return new Metrics(kc);
}
getCustomObjectsClient(clusterDetails: ClusterDetails): CustomObjectsApi {
const kc = this.getKubeConfig(clusterDetails);
return kc.makeApiClient(CustomObjectsApi);
}
}
plugins/kubernetes-backend/src/service/KubernetesFetcher.test.ts
-3
View File
@@ -16,7 +16,6 @@
import { getVoidLogger } from '@backstage/backend-common';
import { KubernetesClientBasedFetcher } from './KubernetesFetcher';
import { KubernetesClientProvider } from './KubernetesClientProvider';
import { ObjectToFetch } from '../types/types';
import {
MockedRequest,
@@ -167,7 +166,6 @@ describe('KubernetesFetcher', () => {
beforeEach(() => {
sut = new KubernetesClientBasedFetcher({
kubernetesClientProvider: new KubernetesClientProvider(),
logger,
});
});
@@ -827,7 +825,6 @@ describe('KubernetesFetcher', () => {
beforeEach(() => {
sut = new KubernetesClientBasedFetcher({
kubernetesClientProvider: new KubernetesClientProvider(),
logger: getVoidLogger(),
});
});
plugins/kubernetes-backend/src/service/KubernetesFetcher.ts
+129 -114
View File
@@ -16,7 +16,9 @@
import {
Cluster,
CoreV1Api,
KubeConfig,
Metrics,
User,
bufferFromFileOrString,
topPods,
@@ -27,9 +29,7 @@ import {
ClusterDetails,
FetchResponseWrapper,
KubernetesFetcher,
KubernetesObjectTypes,
ObjectFetchParams,
ObjectToFetch,
} from '../types/types';
import {
FetchResponse,
@@ -37,13 +37,11 @@ import {
KubernetesErrorTypes,
PodStatusFetchResponse,
} from '@backstage/plugin-kubernetes-common';
import { KubernetesClientProvider } from './KubernetesClientProvider';
import fetch, { Headers, RequestInit } from 'node-fetch';
import fetch, { RequestInit, Response } from 'node-fetch';
import * as https from 'https';
import fs from 'fs-extra';
export interface KubernetesClientBasedFetcherOptions {
kubernetesClientProvider: KubernetesClientProvider;
logger: Logger;
}
@@ -81,14 +79,9 @@ const statusCodeToErrorType = (statusCode: number): KubernetesErrorTypes => {
};
export class KubernetesClientBasedFetcher implements KubernetesFetcher {
private readonly kubernetesClientProvider: KubernetesClientProvider;
private readonly logger: Logger;
constructor({
kubernetesClientProvider,
logger,
}: KubernetesClientBasedFetcherOptions) {
this.kubernetesClientProvider = kubernetesClientProvider;
constructor({ logger }: KubernetesClientBasedFetcherOptions) {
this.logger = logger;
}
@@ -97,16 +90,27 @@ export class KubernetesClientBasedFetcher implements KubernetesFetcher {
): Promise<FetchResponseWrapper> {
const fetchResults = Array.from(params.objectTypesToFetch)
.concat(params.customResources)
.map(toFetch => {
return this.fetchResource(
.map(({ objectType, group, apiVersion, plural }) =>
this.fetchResource(
params.clusterDetails,
toFetch,
group,
apiVersion,
plural,
params.namespace,
params.labelSelector ||
`backstage.io/kubernetes-id=${params.serviceId}`,
toFetch.objectType,
params.namespace,
).catch(this.captureKubernetesErrorsRethrowOthers.bind(this));
});
).then(
(r: Response): Promise<FetchResult> =>
r.ok
? r.json().then(
({ items }): FetchResponse => ({
type: objectType,
resources: items,
}),
)
: this.handleUnsuccessfulResponse(r),
),
);
return Promise.all(fetchResults).then(fetchResultsToResponseWrapper);
}
@@ -115,51 +119,65 @@ export class KubernetesClientBasedFetcher implements KubernetesFetcher {
clusterDetails: ClusterDetails,
namespaces: Set<string>,
): Promise<FetchResponseWrapper> {
const metricsClient =
this.kubernetesClientProvider.getMetricsClient(clusterDetails);
const coreApi =
this.kubernetesClientProvider.getCoreClientByClusterDetails(
clusterDetails,
);
const fetchResults = Array.from(namespaces).map(ns =>
topPods(coreApi, metricsClient, ns)
.then(r => {
return {
const fetchResults = Array.from(namespaces).map(async ns => {
const [podMetrics, podList] = await Promise.all([
this.fetchResource(
clusterDetails,
'metrics.k8s.io',
'v1beta1',
'pods',
ns,
),
this.fetchResource(clusterDetails, '', 'v1', 'pods', ns),
]);
if (podMetrics.ok && podList.ok) {
return topPods(
{
listPodForAllNamespaces: () =>
podList.json().then(b => ({ body: b })),
} as unknown as CoreV1Api,
{
getPodMetrics: () => podMetrics.json(),
} as unknown as Metrics,
).then(
(resources): PodStatusFetchResponse => ({
type: 'podstatus',
resources: r,
} as PodStatusFetchResponse;
})
.catch(this.captureKubernetesErrorsRethrowOthers.bind(this)),
);
resources,
}),
);
} else if (podMetrics.ok) {
return this.handleUnsuccessfulResponse(podList);
}
return this.handleUnsuccessfulResponse(podMetrics);
});
return Promise.all(fetchResults).then(fetchResultsToResponseWrapper);
}
private captureKubernetesErrorsRethrowOthers(e: any): KubernetesFetchError {
if (e.response && e.response.statusCode) {
this.logger.warn(
`statusCode=${e.response.statusCode} for resource ${
e.response.request.uri.pathname
} body=[${JSON.stringify(e.response.body)}]`,
);
return {
errorType: statusCodeToErrorType(e.response.statusCode),
statusCode: e.response.statusCode,
resourcePath: e.response.request.uri.pathname,
};
}
throw e;
private async handleUnsuccessfulResponse(
res: Response,
): Promise<KubernetesFetchError> {
const resourcePath = new URL(res.url).pathname;
this.logger.warn(
`statusCode=${
res.status
} for resource ${resourcePath} body=[${await res.text()}]`,
);
return {
errorType: statusCodeToErrorType(res.status),
statusCode: res.status,
resourcePath,
};
}
private fetchResource(
clusterDetails: ClusterDetails,
resource: ObjectToFetch,
labelSelector: string,
objectType: KubernetesObjectTypes,
group: string,
apiVersion: string,
plural: string,
namespace?: string,
): Promise<FetchResult> {
const { group, apiVersion, plural } = resource;
labelSelector?: string,
): Promise<Response> {
const encode = (s: string) => encodeURIComponent(s);
let resourcePath = group
? `/apis/${encode(group)}/${encode(apiVersion)}`
@@ -169,71 +187,68 @@ export class KubernetesClientBasedFetcher implements KubernetesFetcher {
}
resourcePath += `/${encode(plural)}`;
const headers: Headers = new Headers({
Accept: 'application/json',
'Content-Type': 'application/json',
});
const fetchOptions: RequestInit = {
method: 'GET',
};
let token: Buffer | string;
let url: URL;
if (clusterDetails.serviceAccountToken) {
url = new URL(`${clusterDetails.url}${resourcePath}`);
const [url, requestInit]: [URL, RequestInit] =
clusterDetails.serviceAccountToken
? this.fetchArgsFromClusterDetails(clusterDetails)
: this.fetchArgsInCluster();
if (url.protocol === 'https:') {
fetchOptions.agent = new https.Agent({
ca:
bufferFromFileOrString(
clusterDetails.caFile,
clusterDetails.caData,
) ?? undefined,
rejectUnauthorized: !clusterDetails.skipTLSVerify,
});
}
token = clusterDetails.serviceAccountToken;
} else {
const kc = new KubeConfig();
kc.loadFromCluster();
// loadFromCluster never fails (unless an exception is thrown) and is
// guaranteed to populate the cluster/user/context correctly
const cluster = kc.getCurrentCluster() as Cluster;
const user = kc.getCurrentUser() as User;
url = new URL(`${cluster.server}${resourcePath}`);
if (url.protocol === 'https:') {
fetchOptions.agent = new https.Agent({
ca: fs.readFileSync(cluster.caFile as string),
});
}
token = fs.readFileSync(user.authProvider.config.tokenFile);
url.pathname = resourcePath;
if (labelSelector) {
url.search = `labelSelector=${labelSelector}`;
}
headers.set('Authorization', `Bearer ${token}`);
fetchOptions.headers = headers;
url.search = `labelSelector=${labelSelector}`;
return fetch(url, requestInit);
}
return fetch(url.toString(), fetchOptions).then(r => {
return r.json().then(j => {
if (r.ok) {
return {
type: objectType,
resources: j.items,
};
}
this.logger.warn(
`statusCode=${
r.status
} for resource ${resourcePath} body=[${JSON.stringify(j)}]`,
);
return {
errorType: statusCodeToErrorType(r.status),
statusCode: r.status,
resourcePath,
};
private fetchArgsFromClusterDetails(
clusterDetails: ClusterDetails,
): [URL, RequestInit] {
const requestInit: RequestInit = {
method: 'GET',
headers: {
Accept: 'application/json',
'Content-Type': 'application/json',
Authorization: `Bearer ${clusterDetails.serviceAccountToken}`,
},
};
const url: URL = new URL(clusterDetails.url);
if (url.protocol === 'https:') {
requestInit.agent = new https.Agent({
ca:
bufferFromFileOrString(
clusterDetails.caFile,
clusterDetails.caData,
) ?? undefined,
rejectUnauthorized: !clusterDetails.skipTLSVerify,
});
});
}
return [url, requestInit];
}
private fetchArgsInCluster(): [URL, RequestInit] {
const kc = new KubeConfig();
kc.loadFromCluster();
// loadFromCluster is guaranteed to populate the cluster/user/context
const cluster = kc.getCurrentCluster() as Cluster;
const user = kc.getCurrentUser() as User;
const token = fs.readFileSync(user.authProvider.config.tokenFile);
const requestInit: RequestInit = {
method: 'GET',
headers: {
Accept: 'application/json',
'Content-Type': 'application/json',
Authorization: `Bearer ${token}`,
},
};
const url = new URL(cluster.server);
if (url.protocol === 'https:') {
requestInit.agent = new https.Agent({
ca: fs.readFileSync(cluster.caFile as string),
});
}
return [url, requestInit];
}
}
plugins/kubernetes-backend/src/service/index.ts
-1
View File
@@ -15,7 +15,6 @@
*/
export * from './KubernetesBuilder';
export * from './KubernetesClientProvider';
export { DEFAULT_OBJECTS } from './KubernetesFanOutHandler';
export { HEADER_KUBERNETES_CLUSTER, KubernetesProxy } from './KubernetesProxy';
export * from './router';