Kubernetes: lookup cluster config from google api when using GKE (#4844)

* Restructure config; add GKE cluster locator

Signed-off-by: mclarke <mclarke@spotify.com>

* PR feedback

Signed-off-by: mclarke <mclarke@spotify.com>

* fix region typo

Signed-off-by: mclarke <mclarke@spotify.com>

* replace config api call with clusters endpoint

Signed-off-by: mclarke <mclarke@spotify.com>

* missed tsc error

Signed-off-by: mclarke <mclarke@spotify.com>

* doc tweak

Signed-off-by: mclarke <mclarke@spotify.com>
This commit is contained in:
Matthew Clarke
2021-03-08 10:27:09 +00:00
committed by GitHub
parent 8e15b19a30
commit 9581ff0b4f
20 changed files with 710 additions and 142 deletions
+1
View File
@@ -35,6 +35,7 @@
"@backstage/backend-common": "^0.5.5",
"@backstage/catalog-model": "^0.7.3",
"@backstage/config": "^0.1.3",
"@google-cloud/container": "^2.2.0",
"@kubernetes/client-node": "^0.13.2",
"@types/express": "^4.17.6",
"aws4": "^1.11.0",
+7 -8
View File
@@ -13,15 +13,14 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { ClusterLocatorMethod } from './src/types';
export interface Config {
kubernetes?: {
serviceLocatorMethod: 'multiTenant';
clusterLocatorMethods: 'config'[];
clusters: {
url: string;
name: string;
serviceAccountToken: string | undefined;
authProvider: 'aws' | 'google' | 'serviceAccount';
}[];
serviceLocatorMethod: {
type: 'multiTenant';
};
clusterLocatorMethods: ClusterLocatorMethod[];
};
}
@@ -24,9 +24,7 @@ describe('ConfigClusterLocator', () => {
clusters: [],
});
const sut = ConfigClusterLocator.fromConfig(
config.getConfigArray('clusters'),
);
const sut = ConfigClusterLocator.fromConfig(config);
const result = await sut.getClusters();
@@ -44,9 +42,7 @@ describe('ConfigClusterLocator', () => {
],
});
const sut = ConfigClusterLocator.fromConfig(
config.getConfigArray('clusters'),
);
const sut = ConfigClusterLocator.fromConfig(config);
const result = await sut.getClusters();
@@ -77,9 +73,7 @@ describe('ConfigClusterLocator', () => {
],
});
const sut = ConfigClusterLocator.fromConfig(
config.getConfigArray('clusters'),
);
const sut = ConfigClusterLocator.fromConfig(config);
const result = await sut.getClusters();
@@ -24,11 +24,11 @@ export class ConfigClusterLocator implements KubernetesClustersSupplier {
this.clusterDetails = clusterDetails;
}
static fromConfig(config: Config[]): ConfigClusterLocator {
static fromConfig(config: Config): ConfigClusterLocator {
// TODO: Add validation that authProvider is required and serviceAccountToken
// is required if authProvider is serviceAccount
return new ConfigClusterLocator(
config.map(c => {
config.getConfigArray('clusters').map(c => {
return {
name: c.getString('name'),
url: c.getString('url'),
@@ -0,0 +1,221 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import '@backstage/backend-common';
import { ConfigReader, Config } from '@backstage/config';
import { GkeClusterLocator } from './GkeClusterLocator';
const mockedListClusters = jest.fn();
describe('GkeClusterLocator', () => {
beforeEach(() => {
mockedListClusters.mockRestore();
});
describe('config-parsing', () => {
it('should accept missing region', async () => {
const config: Config = new ConfigReader({
type: 'gke',
projectId: 'some-project',
});
GkeClusterLocator.fromConfigWithClient(config, {
listClusters: mockedListClusters,
} as any);
expect(mockedListClusters).toBeCalledTimes(0);
});
it('should not accept missing projectId', async () => {
const config: Config = new ConfigReader({
type: 'gke',
});
expect(() =>
GkeClusterLocator.fromConfigWithClient(config, {
listClusters: mockedListClusters,
} as any),
).toThrow("Missing required config value at 'projectId'");
expect(mockedListClusters).toBeCalledTimes(0);
});
});
describe('listClusters', () => {
it('empty clusters returns empty cluster details', async () => {
mockedListClusters.mockReturnValueOnce([
{
clusters: [],
},
]);
const config: Config = new ConfigReader({
type: 'gke',
projectId: 'some-project',
region: 'some-region',
});
const sut = GkeClusterLocator.fromConfigWithClient(config, {
listClusters: mockedListClusters,
} as any);
const result = await sut.getClusters();
expect(result).toStrictEqual([]);
expect(mockedListClusters).toBeCalledTimes(1);
expect(mockedListClusters).toHaveBeenCalledWith({
parent: 'projects/some-project/locations/some-region',
});
});
it('1 cluster returns 1 cluster details', async () => {
mockedListClusters.mockReturnValueOnce([
{
clusters: [
{
name: 'some-cluster',
endpoint: '1.2.3.4',
},
],
},
]);
const config: Config = new ConfigReader({
type: 'gke',
projectId: 'some-project',
region: 'some-region',
});
const sut = GkeClusterLocator.fromConfigWithClient(config, {
listClusters: mockedListClusters,
} as any);
const result = await sut.getClusters();
expect(result).toStrictEqual([
{
authProvider: 'google',
name: 'some-cluster',
url: 'https://1.2.3.4',
},
]);
expect(mockedListClusters).toBeCalledTimes(1);
expect(mockedListClusters).toHaveBeenCalledWith({
parent: 'projects/some-project/locations/some-region',
});
});
it('use region wildcard when no region provided', async () => {
mockedListClusters.mockReturnValueOnce([
{
clusters: [
{
name: 'some-cluster',
endpoint: '1.2.3.4',
},
],
},
]);
const config: Config = new ConfigReader({
type: 'gke',
projectId: 'some-project',
});
const sut = GkeClusterLocator.fromConfigWithClient(config, {
listClusters: mockedListClusters,
} as any);
const result = await sut.getClusters();
expect(result).toStrictEqual([
{
authProvider: 'google',
name: 'some-cluster',
url: 'https://1.2.3.4',
},
]);
expect(mockedListClusters).toBeCalledTimes(1);
expect(mockedListClusters).toHaveBeenCalledWith({
parent: 'projects/some-project/locations/-',
});
});
it('2 cluster returns 2 cluster details', async () => {
mockedListClusters.mockReturnValueOnce([
{
clusters: [
{
name: 'some-cluster',
endpoint: '1.2.3.4',
},
{
name: 'some-other-cluster',
endpoint: '6.7.8.9',
},
],
},
]);
const config: Config = new ConfigReader({
type: 'gke',
projectId: 'some-project',
region: 'some-region',
});
const sut = GkeClusterLocator.fromConfigWithClient(config, {
listClusters: mockedListClusters,
} as any);
const result = await sut.getClusters();
expect(result).toStrictEqual([
{
authProvider: 'google',
name: 'some-cluster',
url: 'https://1.2.3.4',
},
{
authProvider: 'google',
name: 'some-other-cluster',
url: 'https://6.7.8.9',
},
]);
expect(mockedListClusters).toBeCalledTimes(1);
expect(mockedListClusters).toHaveBeenCalledWith({
parent: 'projects/some-project/locations/some-region',
});
});
it('Handle errors gracefully', async () => {
mockedListClusters.mockImplementation(() => {
throw new Error('some error');
});
const config: Config = new ConfigReader({
type: 'gke',
projectId: 'some-project',
region: 'some-region',
});
const sut = GkeClusterLocator.fromConfigWithClient(config, {
listClusters: mockedListClusters,
} as any);
await expect(sut.getClusters()).rejects.toThrow(
'There was an error retrieving clusters from GKE for projectId=some-project region=some-region : [some error]',
);
expect(mockedListClusters).toBeCalledTimes(1);
expect(mockedListClusters).toHaveBeenCalledWith({
parent: 'projects/some-project/locations/some-region',
});
});
});
});
@@ -0,0 +1,72 @@
/*
* Copyright 2021 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { ClusterDetails, KubernetesClustersSupplier } from '..';
import { Config } from '@backstage/config';
import * as container from '@google-cloud/container';
export class GkeClusterLocator implements KubernetesClustersSupplier {
private readonly projectId: string;
private readonly region: string | undefined;
private readonly client: container.v1.ClusterManagerClient;
constructor(
projectId: string,
client: container.v1.ClusterManagerClient,
region?: string,
) {
this.projectId = projectId;
this.region = region;
this.client = client;
}
static fromConfigWithClient(
config: Config,
client: container.v1.ClusterManagerClient,
): GkeClusterLocator {
const projectId = config.getString('projectId');
const region = config.getOptionalString('region');
return new GkeClusterLocator(projectId, client, region);
}
static fromConfig(config: Config): GkeClusterLocator {
return GkeClusterLocator.fromConfigWithClient(
config,
new container.v1.ClusterManagerClient(),
);
}
async getClusters(): Promise<ClusterDetails[]> {
const region = this.region ?? '-';
const request = {
parent: `projects/${this.projectId}/locations/${region}`,
};
try {
const [response] = await this.client.listClusters(request);
return (response.clusters ?? []).map(r => ({
// TODO filter out clusters which don't have name or endpoint
name: r.name ?? 'unknown',
url: `https://${r.endpoint ?? ''}`,
authProvider: 'google',
}));
} catch (e) {
throw new Error(
`There was an error retrieving clusters from GKE for projectId=${this.projectId} region=${region} : [${e.message}]`,
);
}
}
}
@@ -22,17 +22,22 @@ describe('getCombinedClusterDetails', () => {
const config: Config = new ConfigReader(
{
kubernetes: {
clusters: [
clusterLocatorMethods: [
{
name: 'cluster1',
serviceAccountToken: 'token',
url: 'http://localhost:8080',
authProvider: 'serviceAccount',
},
{
name: 'cluster2',
url: 'http://localhost:8081',
authProvider: 'google',
type: 'config',
clusters: [
{
name: 'cluster1',
serviceAccountToken: 'token',
url: 'http://localhost:8080',
authProvider: 'serviceAccount',
},
{
name: 'cluster2',
url: 'http://localhost:8081',
authProvider: 'google',
},
],
},
],
},
@@ -40,7 +45,7 @@ describe('getCombinedClusterDetails', () => {
'ctx',
);
const result = await getCombinedClusterDetails(['config'], config);
const result = await getCombinedClusterDetails(config);
expect(result).toStrictEqual([
{
@@ -59,9 +64,36 @@ describe('getCombinedClusterDetails', () => {
});
it('throws an error when using an unsupported cluster locator', async () => {
await expect(
getCombinedClusterDetails(['magic' as any], new ConfigReader({})),
).rejects.toStrictEqual(
const config: Config = new ConfigReader(
{
kubernetes: {
clusterLocatorMethods: [
{
type: 'config',
clusters: [
{
name: 'cluster1',
serviceAccountToken: 'token',
url: 'http://localhost:8080',
authProvider: 'serviceAccount',
},
{
name: 'cluster2',
url: 'http://localhost:8081',
authProvider: 'google',
},
],
},
{
type: 'magic',
},
],
},
},
'ctx',
);
await expect(getCombinedClusterDetails(config)).rejects.toStrictEqual(
new Error('Unsupported kubernetes.clusterLocatorMethods: "magic"'),
);
});
@@ -14,29 +14,34 @@
* limitations under the License.
*/
import { ClusterDetails, ClusterLocatorMethod } from '..';
import { ClusterDetails } from '..';
import { Config } from '@backstage/config';
import { ConfigClusterLocator } from './ConfigClusterLocator';
export { ConfigClusterLocator } from './ConfigClusterLocator';
import { GkeClusterLocator } from './GkeClusterLocator';
export const getCombinedClusterDetails = async (
clusterLocatorMethods: ClusterLocatorMethod[],
rootConfig: Config,
): Promise<ClusterDetails[]> => {
return Promise.all(
clusterLocatorMethods.map(clusterLocatorMethod => {
switch (clusterLocatorMethod) {
case 'config':
return ConfigClusterLocator.fromConfig(
rootConfig.getConfigArray('kubernetes.clusters'),
).getClusters();
default:
throw new Error(
`Unsupported kubernetes.clusterLocatorMethods: "${clusterLocatorMethod}"`,
);
}
}),
rootConfig
.getConfigArray('kubernetes.clusterLocatorMethods')
.map(clusterLocatorMethod => {
const type = clusterLocatorMethod.getString('type');
switch (type) {
case 'config':
return ConfigClusterLocator.fromConfig(
clusterLocatorMethod,
).getClusters();
case 'gke':
return GkeClusterLocator.fromConfig(
clusterLocatorMethod,
).getClusters();
default:
throw new Error(
`Unsupported kubernetes.clusterLocatorMethods: "${type}"`,
);
}
}),
)
.then(res => {
return res.flat();
@@ -29,7 +29,19 @@ describe('router', () => {
getKubernetesObjectsByEntity: jest.fn(),
} as any;
const router = makeRouter(getVoidLogger(), kubernetesFanOutHandler);
const router = makeRouter(getVoidLogger(), kubernetesFanOutHandler, [
{
name: 'some-cluster',
authProvider: 'serviceAccount',
url: 'https://localhost:1234',
serviceAccountToken: 'someToken',
},
{
name: 'some-other-cluster',
url: 'https://localhost:1235',
authProvider: 'google',
},
]);
app = express().use(router);
});
@@ -37,6 +49,25 @@ describe('router', () => {
jest.resetAllMocks();
});
describe('get /clusters', () => {
it('happy path: lists clusters', async () => {
const response = await request(app).get('/clusters');
expect(response.status).toEqual(200);
expect(response.body).toStrictEqual({
items: [
{
name: 'some-cluster',
authProvider: 'serviceAccount',
},
{
name: 'some-other-cluster',
authProvider: 'google',
},
],
});
});
});
describe('post /services/:serviceId', () => {
it('happy path: lists kubernetes objects without auth in request body', async () => {
const result = {
@@ -25,7 +25,6 @@ import {
KubernetesRequestBody,
KubernetesServiceLocator,
ServiceLocatorMethod,
ClusterLocatorMethod,
ClusterDetails,
KubernetesClustersSupplier,
} from '..';
@@ -43,7 +42,7 @@ const getServiceLocator = (
clusterDetails: ClusterDetails[],
): KubernetesServiceLocator => {
const serviceLocatorMethod = config.getString(
'kubernetes.serviceLocatorMethod',
'kubernetes.serviceLocatorMethod.type',
) as ServiceLocatorMethod;
switch (serviceLocatorMethod) {
@@ -61,6 +60,7 @@ const getServiceLocator = (
export const makeRouter = (
logger: Logger,
kubernetesFanOutHandler: KubernetesFanOutHandler,
clusterDetails: ClusterDetails[],
): express.Router => {
const router = Router();
router.use(express.json());
@@ -81,6 +81,14 @@ export const makeRouter = (
}
});
router.get('/clusters', async (_, res) => {
res.json({
items: clusterDetails.map(cd => ({
name: cd.name,
authProvider: cd.authProvider,
})),
});
});
return router;
};
@@ -96,21 +104,18 @@ export async function createRouter(
logger,
});
const clusterLocatorMethods = options.config.getStringArray(
'kubernetes.clusterLocatorMethods',
) as ClusterLocatorMethod[];
let clusterDetails: ClusterDetails[];
if (options.clusterSupplier) {
clusterDetails = await options.clusterSupplier.getClusters();
} else {
clusterDetails = await getCombinedClusterDetails(
clusterLocatorMethods,
options.config,
);
clusterDetails = await getCombinedClusterDetails(options.config);
}
logger.info(
`action=loadClusterDetails numOfClustersLoaded=${clusterDetails.length}`,
);
const serviceLocator = getServiceLocator(options.config, clusterDetails);
const kubernetesFanOutHandler = new KubernetesFanOutHandler(
@@ -119,5 +124,5 @@ export async function createRouter(
serviceLocator,
);
return makeRouter(logger, kubernetesFanOutHandler);
return makeRouter(logger, kubernetesFanOutHandler, clusterDetails);
}
+44 -1
View File
@@ -146,6 +146,49 @@ export interface KubernetesFetchError {
resourcePath?: string;
}
export interface ConfigClusterLocatorMethod {
/**
* @visibility frontend
*/
type: 'config';
clusters: {
/**
* @visibility frontend
*/
url: string;
/**
* @visibility frontend
*/
name: string;
/**
* @visibility secret
*/
serviceAccountToken: string | undefined;
/**
* @visibility frontend
*/
authProvider: 'aws' | 'google' | 'serviceAccount';
}[];
}
export interface GKEClusterLocatorMethod {
/**
* @visibility frontend
*/
type: 'gke';
/**
* @visibility frontend
*/
projectId: string;
/**
* @visibility frontend
*/
region?: string;
}
export type ClusterLocatorMethod =
| ConfigClusterLocatorMethod
| GKEClusterLocatorMethod;
export type ServiceLocatorMethod = 'multiTenant' | 'http'; // TODO implement http
export type ClusterLocatorMethod = 'config';
export type AuthProviderType = 'google' | 'serviceAccount' | 'aws';