From 290eff6692aa3a72beb306566fe2052fbc6a73eb Mon Sep 17 00:00:00 2001 From: Matthew Clarke Date: Thu, 27 Jul 2023 15:09:23 -0400 Subject: [PATCH] feat: GKE Resource provider (#18759) * feat: GKE Resource provider Signed-off-by: Matthew Clarke * feat: configurable parents Signed-off-by: Matthew Clarke * chore: changeset Signed-off-by: Matthew Clarke * test: happy path test Signed-off-by: Matthew Clarke * chore: typos Signed-off-by: Matthew Clarke * docs: api-reports Signed-off-by: Matthew Clarke * docs: readme Signed-off-by: Matthew Clarke --------- Signed-off-by: Matthew Clarke --- .changeset/tough-dolphins-care.md | 5 + .../catalog-backend-module-gcp/.eslintrc.js | 1 + plugins/catalog-backend-module-gcp/README.md | 41 +++ .../alpha-api-report.md | 7 + .../catalog-backend-module-gcp/api-report.md | 48 ++++ .../catalog-backend-module-gcp/config.d.ts | 45 ++++ .../catalog-backend-module-gcp/package.json | 66 +++++ .../catalog-backend-module-gcp/src/alpha.ts | 23 ++ .../catalog-backend-module-gcp/src/index.ts | 24 ++ .../catalogModuleGcpGkeEntityProvider.ts | 52 ++++ .../src/module/index.ts | 17 ++ .../src/providers/GkeEntityProvider.test.ts | 251 ++++++++++++++++++ .../src/providers/GkeEntityProvider.ts | 223 ++++++++++++++++ .../src/providers/index.ts | 17 ++ .../src/setupTests.ts | 17 ++ yarn.lock | 25 +- 16 files changed, 858 insertions(+), 4 deletions(-) create mode 100644 .changeset/tough-dolphins-care.md create mode 100644 plugins/catalog-backend-module-gcp/.eslintrc.js create mode 100644 plugins/catalog-backend-module-gcp/README.md create mode 100644 plugins/catalog-backend-module-gcp/alpha-api-report.md create mode 100644 plugins/catalog-backend-module-gcp/api-report.md create mode 100644 plugins/catalog-backend-module-gcp/config.d.ts create mode 100644 plugins/catalog-backend-module-gcp/package.json create mode 100644 plugins/catalog-backend-module-gcp/src/alpha.ts create mode 100644 plugins/catalog-backend-module-gcp/src/index.ts create mode 100644 plugins/catalog-backend-module-gcp/src/module/catalogModuleGcpGkeEntityProvider.ts create mode 100644 plugins/catalog-backend-module-gcp/src/module/index.ts create mode 100644 plugins/catalog-backend-module-gcp/src/providers/GkeEntityProvider.test.ts create mode 100644 plugins/catalog-backend-module-gcp/src/providers/GkeEntityProvider.ts create mode 100644 plugins/catalog-backend-module-gcp/src/providers/index.ts create mode 100644 plugins/catalog-backend-module-gcp/src/setupTests.ts diff --git a/.changeset/tough-dolphins-care.md b/.changeset/tough-dolphins-care.md new file mode 100644 index 0000000000..7d9739ec10 --- /dev/null +++ b/.changeset/tough-dolphins-care.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend-module-gcp': minor +--- + +Added GCP catalog plugin with GKE provider diff --git a/plugins/catalog-backend-module-gcp/.eslintrc.js b/plugins/catalog-backend-module-gcp/.eslintrc.js new file mode 100644 index 0000000000..e2a53a6ad2 --- /dev/null +++ b/plugins/catalog-backend-module-gcp/.eslintrc.js @@ -0,0 +1 @@ +module.exports = require('@backstage/cli/config/eslint-factory')(__dirname); diff --git a/plugins/catalog-backend-module-gcp/README.md b/plugins/catalog-backend-module-gcp/README.md new file mode 100644 index 0000000000..d1ef926310 --- /dev/null +++ b/plugins/catalog-backend-module-gcp/README.md @@ -0,0 +1,41 @@ +# Catalog Backend Module for GCP + +This is an extension module to the plugin-catalog-backend plugin, containing catalog processors and providers to ingest GCP resources as `Resource` kind entities. + +## installation + +Register the plugin in `catalog.ts`` + +```typescript +import { GkeEntityProvider } from '@backstage/plugin-catalog-backend-module-gcp'; + +... + +builder.addEntityProvider( + GkeEntityProvider.fromConfig({ + logger: env.logger, + scheduler: env.scheduler, + config: env.config + }) +); +``` + +Update `app-config.yaml` as follows: + +```yaml +catalog: + providers: + gcp: + gke: + parents: + # consult https://cloud.google.com/kubernetes-engine/docs/ for valid values + # list all clusters in the project + - 'projects/some-project/locations/-' + # list all clusters in the region, in the project + - 'projects/some-other-project/locations/some-region' + schedule: # optional; same options as in TaskScheduleDefinition + # supports cron, ISO duration, "human duration" as used in code + frequency: { minutes: 30 } + # supports ISO duration, "human duration" as used in code + timeout: { minutes: 3 } +``` diff --git a/plugins/catalog-backend-module-gcp/alpha-api-report.md b/plugins/catalog-backend-module-gcp/alpha-api-report.md new file mode 100644 index 0000000000..6e6f0427e9 --- /dev/null +++ b/plugins/catalog-backend-module-gcp/alpha-api-report.md @@ -0,0 +1,7 @@ +## API Report File for "@backstage/plugin-catalog-backend-module-gcp" + +> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). + +```ts + +``` diff --git a/plugins/catalog-backend-module-gcp/api-report.md b/plugins/catalog-backend-module-gcp/api-report.md new file mode 100644 index 0000000000..6f963ed51d --- /dev/null +++ b/plugins/catalog-backend-module-gcp/api-report.md @@ -0,0 +1,48 @@ +## API Report File for "@backstage/plugin-catalog-backend-module-gcp" + +> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). + +```ts +import { BackendFeature } from '@backstage/backend-plugin-api'; +import { Config } from '@backstage/config'; +import * as container from '@google-cloud/container'; +import { EntityProvider } from '@backstage/plugin-catalog-node'; +import { EntityProviderConnection } from '@backstage/plugin-catalog-node'; +import { Logger } from 'winston'; +import { SchedulerService } from '@backstage/backend-plugin-api'; + +// @public +export const catalogModuleGcpGkeEntityProvider: () => BackendFeature; + +// @public +export class GkeEntityProvider implements EntityProvider { + // (undocumented) + connect(connection: EntityProviderConnection): Promise; + // (undocumented) + static fromConfig({ + logger, + scheduler, + config, + }: { + logger: Logger; + scheduler: SchedulerService; + config: Config; + }): GkeEntityProvider; + // (undocumented) + static fromConfigWithClient({ + logger, + scheduler, + config, + clusterManagerClient, + }: { + logger: Logger; + scheduler: SchedulerService; + config: Config; + clusterManagerClient: container.v1.ClusterManagerClient; + }): GkeEntityProvider; + // (undocumented) + getProviderName(): string; + // (undocumented) + refresh(): Promise; +} +``` diff --git a/plugins/catalog-backend-module-gcp/config.d.ts b/plugins/catalog-backend-module-gcp/config.d.ts new file mode 100644 index 0000000000..d6884d32ce --- /dev/null +++ b/plugins/catalog-backend-module-gcp/config.d.ts @@ -0,0 +1,45 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { TaskScheduleDefinitionConfig } from '@backstage/backend-tasks'; + +export interface Config { + catalog?: { + /** + * List of provider-specific options and attributes + */ + providers?: { + /** + * GCPCatalogModuleConfig configuration + */ + gcp?: { + /** + * Config for GKE clusters + */ + gke?: { + /** + * Locations to list clusters from + */ + parents: string[]; + /** + * (Optional) TaskScheduleDefinition for the refresh. + */ + schedule: TaskScheduleDefinitionConfig; + }; + }; + }; + }; +} diff --git a/plugins/catalog-backend-module-gcp/package.json b/plugins/catalog-backend-module-gcp/package.json new file mode 100644 index 0000000000..12285d26eb --- /dev/null +++ b/plugins/catalog-backend-module-gcp/package.json @@ -0,0 +1,66 @@ +{ + "name": "@backstage/plugin-catalog-backend-module-gcp", + "description": "A Backstage catalog backend module that helps integrate towards GCP", + "version": "0.0.0", + "main": "src/index.ts", + "types": "src/index.ts", + "license": "Apache-2.0", + "publishConfig": { + "access": "public" + }, + "exports": { + ".": "./src/index.ts", + "./alpha": "./src/alpha.ts", + "./package.json": "./package.json" + }, + "typesVersions": { + "*": { + "alpha": [ + "src/alpha.ts" + ], + "package.json": [ + "package.json" + ] + } + }, + "backstage": { + "role": "backend-plugin-module" + }, + "homepage": "https://backstage.io", + "repository": { + "type": "git", + "url": "https://github.com/backstage/backstage", + "directory": "plugins/catalog-backend-module-gcp" + }, + "keywords": [ + "backstage" + ], + "scripts": { + "start": "backstage-cli package start", + "build": "backstage-cli package build", + "lint": "backstage-cli package lint", + "test": "backstage-cli package test", + "prepack": "backstage-cli package prepack", + "postpack": "backstage-cli package postpack", + "clean": "backstage-cli package clean" + }, + "dependencies": { + "@backstage/backend-common": "workspace:^", + "@backstage/backend-plugin-api": "workspace:^", + "@backstage/backend-tasks": "workspace:^", + "@backstage/config": "workspace:^", + "@backstage/plugin-catalog-node": "workspace:^", + "@backstage/plugin-kubernetes-common": "workspace:^", + "@google-cloud/container": "^4.15.0", + "winston": "^3.2.1" + }, + "devDependencies": { + "@backstage/backend-test-utils": "workspace:^", + "@backstage/cli": "workspace:^" + }, + "files": [ + "config.d.ts", + "dist" + ], + "configSchema": "config.d.ts" +} diff --git a/plugins/catalog-backend-module-gcp/src/alpha.ts b/plugins/catalog-backend-module-gcp/src/alpha.ts new file mode 100644 index 0000000000..9eec69e76f --- /dev/null +++ b/plugins/catalog-backend-module-gcp/src/alpha.ts @@ -0,0 +1,23 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * A Backstage catalog backend module that helps integrate towards GCP + * + * @packageDocumentation + */ + +export {}; diff --git a/plugins/catalog-backend-module-gcp/src/index.ts b/plugins/catalog-backend-module-gcp/src/index.ts new file mode 100644 index 0000000000..fb3984a04d --- /dev/null +++ b/plugins/catalog-backend-module-gcp/src/index.ts @@ -0,0 +1,24 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * A Backstage catalog backend module that helps integrate towards GCP + * + * @packageDocumentation + */ + +export * from './providers'; +export * from './module'; diff --git a/plugins/catalog-backend-module-gcp/src/module/catalogModuleGcpGkeEntityProvider.ts b/plugins/catalog-backend-module-gcp/src/module/catalogModuleGcpGkeEntityProvider.ts new file mode 100644 index 0000000000..c13c6cbe0e --- /dev/null +++ b/plugins/catalog-backend-module-gcp/src/module/catalogModuleGcpGkeEntityProvider.ts @@ -0,0 +1,52 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { loggerToWinstonLogger } from '@backstage/backend-common'; +import { + coreServices, + createBackendModule, +} from '@backstage/backend-plugin-api'; +import { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node/alpha'; +import { GkeEntityProvider } from '../providers/GkeEntityProvider'; + +/** + * Registers the GcpGkeEntityProvider with the catalog processing extension point. + * + * @public + */ +export const catalogModuleGcpGkeEntityProvider = createBackendModule({ + pluginId: 'catalog', + moduleId: 'gcpGkeEntityProvider', + register(env) { + env.registerInit({ + deps: { + config: coreServices.config, + catalog: catalogProcessingExtensionPoint, + logger: coreServices.logger, + scheduler: coreServices.scheduler, + }, + async init({ config, catalog, logger, scheduler }) { + catalog.addEntityProvider( + GkeEntityProvider.fromConfig({ + logger: loggerToWinstonLogger(logger), + scheduler, + config, + }), + ); + }, + }); + }, +}); diff --git a/plugins/catalog-backend-module-gcp/src/module/index.ts b/plugins/catalog-backend-module-gcp/src/module/index.ts new file mode 100644 index 0000000000..c7cd310c97 --- /dev/null +++ b/plugins/catalog-backend-module-gcp/src/module/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { catalogModuleGcpGkeEntityProvider } from './catalogModuleGcpGkeEntityProvider'; diff --git a/plugins/catalog-backend-module-gcp/src/providers/GkeEntityProvider.test.ts b/plugins/catalog-backend-module-gcp/src/providers/GkeEntityProvider.test.ts new file mode 100644 index 0000000000..20e91a85bf --- /dev/null +++ b/plugins/catalog-backend-module-gcp/src/providers/GkeEntityProvider.test.ts @@ -0,0 +1,251 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { GkeEntityProvider } from './GkeEntityProvider'; +import { TaskRunner } from '@backstage/backend-tasks'; +import { + ANNOTATION_KUBERNETES_API_SERVER, + ANNOTATION_KUBERNETES_API_SERVER_CA, + ANNOTATION_KUBERNETES_AUTH_PROVIDER, +} from '@backstage/plugin-kubernetes-common'; +import * as container from '@google-cloud/container'; +import { ConfigReader } from '@backstage/config'; + +describe('GkeEntityProvider', () => { + const clusterManagerClientMock = { + listClusters: jest.fn(), + }; + const connectionMock = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + const taskRunner = { + createScheduleFn: jest.fn(), + run: jest.fn(), + } as TaskRunner; + const schedulerMock = { + createScheduledTaskRunner: jest.fn(), + } as any; + const logger = { + info: jest.fn(), + error: jest.fn(), + }; + let gkeEntityProvider: GkeEntityProvider; + + beforeEach(async () => { + jest.resetAllMocks(); + schedulerMock.createScheduledTaskRunner.mockReturnValue(taskRunner); + gkeEntityProvider = GkeEntityProvider.fromConfigWithClient({ + logger: logger as any, + config: new ConfigReader({ + catalog: { + providers: { + gcp: { + gke: { + parents: ['parent1', 'parent2'], + schedule: { + frequency: { + minutes: 3, + }, + timeout: { + minutes: 3, + }, + }, + }, + }, + }, + }, + }), + scheduler: schedulerMock, + clusterManagerClient: clusterManagerClientMock as any, + }); + await gkeEntityProvider.connect(connectionMock); + }); + + it('should return clusters as Resources', async () => { + clusterManagerClientMock.listClusters.mockImplementation(req => { + if (req.parent === 'parent1') { + return [ + { + clusters: [ + { + name: 'some-cluster', + endpoint: 'http://127.0.0.1:1234', + location: 'some-location', + selfLink: 'http://127.0.0.1/some-link', + masterAuth: { + clusterCaCertificate: 'abcdefg', + }, + }, + ], + }, + ]; + } else if (req.parent === 'parent2') { + return [ + { + clusters: [ + { + name: 'some-other-cluster', + endpoint: 'http://127.0.0.1:5678', + location: 'some-other-location', + selfLink: 'http://127.0.0.1/some-other-link', + masterAuth: { + // no CA cert is ok + }, + }, + ], + }, + ]; + } + + throw new Error(`unexpected parent ${req.parent}`); + }); + await gkeEntityProvider.refresh(); + expect(connectionMock.applyMutation).toHaveBeenCalledWith({ + type: 'full', + entities: [ + { + locationKey: 'gcp-gke:some-location', + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Resource', + metadata: { + annotations: { + [ANNOTATION_KUBERNETES_API_SERVER]: 'http://127.0.0.1:1234', + [ANNOTATION_KUBERNETES_API_SERVER_CA]: 'abcdefg', + [ANNOTATION_KUBERNETES_AUTH_PROVIDER]: 'google', + 'backstage.io/managed-by-location': 'gcp-gke:some-location', + 'backstage.io/managed-by-origin-location': + 'gcp-gke:some-location', + }, + name: 'some-cluster', + namespace: 'default', + }, + spec: { + type: 'kubernetes-cluster', + owner: 'unknown', + }, + }, + }, + { + locationKey: 'gcp-gke:some-other-location', + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Resource', + metadata: { + annotations: { + [ANNOTATION_KUBERNETES_API_SERVER]: 'http://127.0.0.1:5678', + [ANNOTATION_KUBERNETES_API_SERVER_CA]: '', + [ANNOTATION_KUBERNETES_AUTH_PROVIDER]: 'google', + 'backstage.io/managed-by-location': + 'gcp-gke:some-other-location', + 'backstage.io/managed-by-origin-location': + 'gcp-gke:some-other-location', + }, + name: 'some-other-cluster', + namespace: 'default', + }, + spec: { + type: 'kubernetes-cluster', + owner: 'unknown', + }, + }, + }, + ], + }); + }); + + const ignoredPartialClustersTests: [ + string, + container.protos.google.container.v1.ICluster, + ][] = [ + [ + 'no-cluster-name', + { + endpoint: 'http://127.0.0.1:1234', + location: 'some-location', + selfLink: 'http://127.0.0.1/some-link', + masterAuth: { + clusterCaCertificate: 'abcdefg', + }, + }, + ], + [ + 'no-self-link', + { + // no selfLink + name: 'some-name', + endpoint: 'http://127.0.0.1:1234', + location: 'some-location', + masterAuth: { + clusterCaCertificate: 'abcdefg', + }, + }, + ], + [ + 'no-endpoint', + { + name: 'some-name', + location: 'some-location', + selfLink: 'http://127.0.0.1/some-link', + masterAuth: { + clusterCaCertificate: 'abcdefg', + }, + }, + ], + [ + 'no-location', + { + name: 'some-name', + endpoint: 'http://127.0.0.1:1234', + selfLink: 'http://127.0.0.1/some-link', + masterAuth: { + clusterCaCertificate: 'abcdefg', + }, + }, + ], + ]; + + it.each(ignoredPartialClustersTests)( + 'ignore cluster - %s', + async (_name, ignoredCluster) => { + clusterManagerClientMock.listClusters.mockImplementation(req => { + if (req.parent === 'parent1') { + return [ignoredCluster]; + } + return [ + { + clusters: [], + }, + ]; + }); + await gkeEntityProvider.refresh(); + expect(connectionMock.applyMutation).toHaveBeenCalledWith({ + type: 'full', + entities: [], + }); + }, + ); + + it('should log GKE API errors', async () => { + clusterManagerClientMock.listClusters.mockRejectedValue( + new Error('some-error'), + ); + await gkeEntityProvider.refresh(); + expect(connectionMock.applyMutation).toHaveBeenCalledTimes(0); + expect(logger.error).toHaveBeenCalledTimes(1); + }); +}); diff --git a/plugins/catalog-backend-module-gcp/src/providers/GkeEntityProvider.ts b/plugins/catalog-backend-module-gcp/src/providers/GkeEntityProvider.ts new file mode 100644 index 0000000000..0130955737 --- /dev/null +++ b/plugins/catalog-backend-module-gcp/src/providers/GkeEntityProvider.ts @@ -0,0 +1,223 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { + TaskRunner, + readTaskScheduleDefinitionFromConfig, +} from '@backstage/backend-tasks'; +import { + DeferredEntity, + EntityProvider, + EntityProviderConnection, +} from '@backstage/plugin-catalog-node'; + +import { Logger } from 'winston'; +import * as container from '@google-cloud/container'; +import { + ANNOTATION_KUBERNETES_API_SERVER, + ANNOTATION_KUBERNETES_API_SERVER_CA, + ANNOTATION_KUBERNETES_AUTH_PROVIDER, +} from '@backstage/plugin-kubernetes-common'; +import { Config } from '@backstage/config'; +import { SchedulerService } from '@backstage/backend-plugin-api'; + +/** + * Catalog provider to ingest GKE clusters + * + * @public + */ +export class GkeEntityProvider implements EntityProvider { + private readonly logger: Logger; + private readonly scheduleFn: () => Promise; + private readonly gkeParents: string[]; + private readonly clusterManagerClient: container.v1.ClusterManagerClient; + private connection?: EntityProviderConnection; + + private constructor( + logger: Logger, + taskRunner: TaskRunner, + gkeParents: string[], + clusterManagerClient: container.v1.ClusterManagerClient, + ) { + this.logger = logger; + this.scheduleFn = this.createScheduleFn(taskRunner); + this.gkeParents = gkeParents; + this.clusterManagerClient = clusterManagerClient; + } + + public static fromConfig({ + logger, + scheduler, + config, + }: { + logger: Logger; + scheduler: SchedulerService; + config: Config; + }) { + return GkeEntityProvider.fromConfigWithClient({ + logger, + scheduler: scheduler, + config, + clusterManagerClient: new container.v1.ClusterManagerClient(), + }); + } + + public static fromConfigWithClient({ + logger, + scheduler, + config, + clusterManagerClient, + }: { + logger: Logger; + scheduler: SchedulerService; + config: Config; + clusterManagerClient: container.v1.ClusterManagerClient; + }) { + const gkeProviderConfig = config.getConfig('catalog.providers.gcp.gke'); + const schedule = readTaskScheduleDefinitionFromConfig( + gkeProviderConfig.getConfig('schedule'), + ); + return new GkeEntityProvider( + logger, + scheduler.createScheduledTaskRunner(schedule), + gkeProviderConfig.getStringArray('parents'), + clusterManagerClient, + ); + } + + getProviderName(): string { + return `gcp-gke`; + } + + async connect(connection: EntityProviderConnection): Promise { + this.connection = connection; + await this.scheduleFn(); + } + + private filterOutUndefinedDeferredEntity( + e: DeferredEntity | undefined, + ): e is DeferredEntity { + return e !== undefined; + } + + private filterOutUndefinedCluster( + c: container.protos.google.container.v1.ICluster | null | undefined, + ): c is container.protos.google.container.v1.ICluster { + return c !== undefined && c !== null; + } + + private clusterToResource( + cluster: container.protos.google.container.v1.ICluster, + ): DeferredEntity | undefined { + const location = `${this.getProviderName()}:${cluster.location}`; + + if (!cluster.name || !cluster.selfLink || !location || !cluster.endpoint) { + this.logger.warn( + `ignoring partial cluster, one of name=${cluster.name}, endpoint=${cluster.endpoint}, selfLink=${cluster.selfLink} or location=${cluster.location} is missing`, + ); + return undefined; + } + + // TODO fix location type + return { + locationKey: location, + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Resource', + metadata: { + annotations: { + [ANNOTATION_KUBERNETES_API_SERVER]: cluster.endpoint, + [ANNOTATION_KUBERNETES_API_SERVER_CA]: + cluster.masterAuth?.clusterCaCertificate || '', + [ANNOTATION_KUBERNETES_AUTH_PROVIDER]: 'google', + 'backstage.io/managed-by-location': location, + 'backstage.io/managed-by-origin-location': location, + }, + name: cluster.name, + namespace: 'default', + }, + spec: { + type: 'kubernetes-cluster', + owner: 'unknown', + }, + }, + }; + } + + private createScheduleFn(taskRunner: TaskRunner): () => Promise { + return async () => { + const taskId = `${this.getProviderName()}:refresh`; + return taskRunner.run({ + id: taskId, + fn: async () => { + try { + await this.refresh(); + } catch (error) { + this.logger.error(error); + } + }, + }); + }; + } + + private async getClusters(): Promise< + container.protos.google.container.v1.ICluster[] + > { + const clusters = await Promise.all( + this.gkeParents.map(async parent => { + const request = { + parent: parent, + }; + const [response] = await this.clusterManagerClient.listClusters( + request, + ); + return response.clusters?.filter(this.filterOutUndefinedCluster) ?? []; + }), + ); + return clusters.flat(); + } + + async refresh() { + if (!this.connection) { + throw new Error('Not initialized'); + } + + this.logger.info('Discovering GKE clusters'); + + let clusters: container.protos.google.container.v1.ICluster[]; + + try { + clusters = await this.getClusters(); + } catch (e) { + this.logger.error('error fetching GKE clusters', e); + return; + } + const resources = + clusters + .map(c => this.clusterToResource(c)) + .filter(this.filterOutUndefinedDeferredEntity) ?? []; + + this.logger.info( + `Ingesting GKE clusters [${resources + .map(r => r.entity.metadata.name) + .join(', ')}]`, + ); + + await this.connection.applyMutation({ + type: 'full', + entities: resources, + }); + } +} diff --git a/plugins/catalog-backend-module-gcp/src/providers/index.ts b/plugins/catalog-backend-module-gcp/src/providers/index.ts new file mode 100644 index 0000000000..7846afc3ed --- /dev/null +++ b/plugins/catalog-backend-module-gcp/src/providers/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { GkeEntityProvider } from './GkeEntityProvider'; diff --git a/plugins/catalog-backend-module-gcp/src/setupTests.ts b/plugins/catalog-backend-module-gcp/src/setupTests.ts new file mode 100644 index 0000000000..aa70772592 --- /dev/null +++ b/plugins/catalog-backend-module-gcp/src/setupTests.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export {}; diff --git a/yarn.lock b/yarn.lock index 3f9b0ee1bc..eec689ba6a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5160,6 +5160,23 @@ __metadata: languageName: unknown linkType: soft +"@backstage/plugin-catalog-backend-module-gcp@workspace:plugins/catalog-backend-module-gcp": + version: 0.0.0-use.local + resolution: "@backstage/plugin-catalog-backend-module-gcp@workspace:plugins/catalog-backend-module-gcp" + dependencies: + "@backstage/backend-common": "workspace:^" + "@backstage/backend-plugin-api": "workspace:^" + "@backstage/backend-tasks": "workspace:^" + "@backstage/backend-test-utils": "workspace:^" + "@backstage/cli": "workspace:^" + "@backstage/config": "workspace:^" + "@backstage/plugin-catalog-node": "workspace:^" + "@backstage/plugin-kubernetes-common": "workspace:^" + "@google-cloud/container": ^4.15.0 + winston: ^3.2.1 + languageName: unknown + linkType: soft + "@backstage/plugin-catalog-backend-module-gerrit@workspace:plugins/catalog-backend-module-gerrit": version: 0.0.0-use.local resolution: "@backstage/plugin-catalog-backend-module-gerrit@workspace:plugins/catalog-backend-module-gerrit" @@ -10912,12 +10929,12 @@ __metadata: languageName: node linkType: hard -"@google-cloud/container@npm:^4.0.0": - version: 4.13.0 - resolution: "@google-cloud/container@npm:4.13.0" +"@google-cloud/container@npm:^4.0.0, @google-cloud/container@npm:^4.15.0": + version: 4.15.0 + resolution: "@google-cloud/container@npm:4.15.0" dependencies: google-gax: ^3.5.8 - checksum: 21e57e8c69e2df8cf6899541dc405f9a6c05d999289cf211c2bb8dc6e33bf7ef36ff255cfde75c6448dd335fb5b707e77f5df49bee561decbd5ed398c58db6f6 + checksum: e81f1b708ab44c55b8e998d34c442baed10a113828ef51b4a84b806a2b04a10a0dc40ee0aa202386ac6158b34b9d3f03467d7008da9e4f709fdb1f021e725b8c languageName: node linkType: hard