diff --git a/.changeset/pink-pets-jump.md b/.changeset/pink-pets-jump.md new file mode 100644 index 0000000000..d9bb97e36e --- /dev/null +++ b/.changeset/pink-pets-jump.md @@ -0,0 +1,5 @@ +--- +'@backstage/backend-defaults': patch +--- + +Support `connection.type: cloudsql` in database client for usage with `@google-cloud/cloud-sql-connector` and `iam` auth diff --git a/packages/backend-defaults/config.d.ts b/packages/backend-defaults/config.d.ts index c15fe45139..8272bc5ca8 100644 --- a/packages/backend-defaults/config.d.ts +++ b/packages/backend-defaults/config.d.ts @@ -384,6 +384,16 @@ export interface Config { */ connection: | string + | { + /** + * The specific config for cloudsql connections + */ + type: 'cloudsql'; + /** + * The instance connection name for the cloudsql instance, e.g. `project:region:instance` + */ + instance: string; + } | { /** * Password that belongs to the client User @@ -441,7 +451,30 @@ export interface Config { * Database connection string or Knex object override * @visibility secret */ - connection?: string | object; + connection?: + | string + | { + /** + * The specific config for cloudsql connections + */ + type: 'cloudsql'; + /** + * The instance connection name for the cloudsql instance, e.g. `project:region:instance` + */ + instance: string; + } + | { + /** + * Password that belongs to the client User + * @visibility secret + */ + password?: string; + /** + * Other connection settings + */ + [key: string]: unknown; + }; + /** * Whether to ensure the given database exists by creating it if it does not. * Defaults to base config if unspecified. diff --git a/packages/backend-defaults/package.json b/packages/backend-defaults/package.json index 689688630e..b0f77d5c44 100644 --- a/packages/backend-defaults/package.json +++ b/packages/backend-defaults/package.json @@ -185,11 +185,20 @@ "yn": "^4.0.0", "zod": "^3.22.4" }, + "peerDependencies": { + "@google-cloud/cloud-sql-connector": "^1.4.0" + }, + "peerDependenciesMeta": { + "@google-cloud/cloud-sql-connector": { + "optional": true + } + }, "devDependencies": { "@aws-sdk/util-stream-node": "^3.350.0", "@backstage/backend-plugin-api": "workspace:^", "@backstage/backend-test-utils": "workspace:^", "@backstage/cli": "workspace:^", + "@google-cloud/cloud-sql-connector": "^1.4.0", "@types/archiver": "^6.0.0", "@types/base64-stream": "^1.0.2", "@types/concat-stream": "^2.0.0", diff --git a/packages/backend-defaults/src/entrypoints/database/connectors/postgres.test.ts b/packages/backend-defaults/src/entrypoints/database/connectors/postgres.test.ts index 1e07984967..6f66f98f15 100644 --- a/packages/backend-defaults/src/entrypoints/database/connectors/postgres.test.ts +++ b/packages/backend-defaults/src/entrypoints/database/connectors/postgres.test.ts @@ -22,6 +22,8 @@ import { parsePgConnectionString, } from './postgres'; +jest.mock('@google-cloud/cloud-sql-connector'); + describe('postgres', () => { const createMockConnection = () => ({ host: 'acme', @@ -37,33 +39,35 @@ describe('postgres', () => { new ConfigReader({ client: 'pg', connection }); describe('buildPgDatabaseConfig', () => { - it('builds a postgres config', () => { + it('builds a postgres config', async () => { const mockConnection = createMockConnection(); - expect(buildPgDatabaseConfig(createConfig(mockConnection))).toEqual({ - client: 'pg', - connection: mockConnection, - useNullAsDefault: true, - }); - }); - - it('builds a connection string config', () => { - const mockConnectionString = createMockConnectionString(); - - expect(buildPgDatabaseConfig(createConfig(mockConnectionString))).toEqual( + expect(await buildPgDatabaseConfig(createConfig(mockConnection))).toEqual( { client: 'pg', - connection: mockConnectionString, + connection: mockConnection, useNullAsDefault: true, }, ); }); - it('overrides the database name', () => { + it('builds a connection string config', async () => { + const mockConnectionString = createMockConnectionString(); + + expect( + await buildPgDatabaseConfig(createConfig(mockConnectionString)), + ).toEqual({ + client: 'pg', + connection: mockConnectionString, + useNullAsDefault: true, + }); + }); + + it('overrides the database name', async () => { const mockConnection = createMockConnection(); expect( - buildPgDatabaseConfig(createConfig(mockConnection), { + await buildPgDatabaseConfig(createConfig(mockConnection), { connection: { database: 'other_db' }, }), ).toEqual({ @@ -76,14 +80,14 @@ describe('postgres', () => { }); }); - it('overrides the schema name', () => { + it('overrides the schema name', async () => { const mockConnection = { ...createMockConnection(), schema: 'schemaName', }; expect( - buildPgDatabaseConfig(createConfig(mockConnection), { + await buildPgDatabaseConfig(createConfig(mockConnection), { searchPath: ['schemaName'], }), ).toEqual({ @@ -94,11 +98,11 @@ describe('postgres', () => { }); }); - it('adds additional config settings', () => { + it('adds additional config settings', async () => { const mockConnection = createMockConnection(); expect( - buildPgDatabaseConfig(createConfig(mockConnection), { + await buildPgDatabaseConfig(createConfig(mockConnection), { connection: { database: 'other_db' }, pool: { min: 0, max: 7 }, debug: true, @@ -115,12 +119,12 @@ describe('postgres', () => { }); }); - it('overrides the database from connection string', () => { + it('overrides the database from connection string', async () => { const mockConnectionString = createMockConnectionString(); const mockConnection = createMockConnection(); expect( - buildPgDatabaseConfig(createConfig(mockConnectionString), { + await buildPgDatabaseConfig(createConfig(mockConnectionString), { connection: { database: 'other_db' }, }), ).toEqual({ @@ -133,6 +137,89 @@ describe('postgres', () => { useNullAsDefault: true, }); }); + + it('uses the correct config when using cloudsql', async () => { + expect( + await buildPgDatabaseConfig( + new ConfigReader({ + client: 'pg', + connection: { + type: 'cloudsql', + user: 'ben@gke.com', + instance: 'project:region:instance', + port: 5423, + }, + }), + { connection: { database: 'other_db' } }, + ), + ).toEqual({ + client: 'pg', + connection: { + user: 'ben@gke.com', + port: 5423, + database: 'other_db', + }, + useNullAsDefault: true, + }); + }); + + it('should throw with incorrect config', async () => { + await expect( + buildPgDatabaseConfig( + new ConfigReader({ + client: 'pg', + connection: { + type: 'cloudsql', + }, + }), + ), + ).rejects.toThrow(/Missing instance connection name for Cloud SQL/); + + await expect( + buildPgDatabaseConfig( + new ConfigReader({ + client: 'not-pg', + connection: { + type: 'cloudsql', + instance: 'asd:asd:asd', + }, + }), + ), + ).rejects.toThrow(/Cloud SQL only supports the pg client/); + }); + + it('adds the settings from cloud-sql-connector', async () => { + const { Connector } = jest.requireMock( + '@google-cloud/cloud-sql-connector', + ) as jest.Mocked; + + const mockStream = (): any => {}; + Connector.prototype.getOptions.mockResolvedValue({ stream: mockStream }); + + expect( + await buildPgDatabaseConfig( + new ConfigReader({ + client: 'pg', + connection: { + type: 'cloudsql', + user: 'ben@gke.com', + instance: 'project:region:instance', + port: 5423, + }, + }), + { connection: { database: 'other_db' } }, + ), + ).toEqual({ + client: 'pg', + connection: { + user: 'ben@gke.com', + port: 5423, + stream: mockStream, + database: 'other_db', + }, + useNullAsDefault: true, + }); + }); }); describe('getPgConnectionConfig', () => { @@ -174,9 +261,9 @@ describe('postgres', () => { }); describe('createPgDatabaseClient', () => { - it('creates a postgres knex instance', () => { + it('creates a postgres knex instance', async () => { expect( - createPgDatabaseClient( + await createPgDatabaseClient( createConfig({ host: 'acme', user: 'foo', @@ -187,14 +274,14 @@ describe('postgres', () => { ).toBeTruthy(); }); - it('attempts to read an ssl cert', () => { - expect(() => + it('attempts to read an ssl cert', async () => { + await expect(() => createPgDatabaseClient( createConfig( 'postgresql://postgres:pass@localhost:5432/dbname?sslrootcert=/path/to/file', ), ), - ).toThrow(/no such file or directory/); + ).rejects.toThrow(/no such file or directory/); }); }); diff --git a/packages/backend-defaults/src/entrypoints/database/connectors/postgres.ts b/packages/backend-defaults/src/entrypoints/database/connectors/postgres.ts index b5cfffd1f4..5f21d278e7 100644 --- a/packages/backend-defaults/src/entrypoints/database/connectors/postgres.ts +++ b/packages/backend-defaults/src/entrypoints/database/connectors/postgres.ts @@ -37,11 +37,11 @@ const ddlLimiter = limiterFactory(1); * @param dbConfig - The database config * @param overrides - Additional options to merge with the config */ -export function createPgDatabaseClient( +export async function createPgDatabaseClient( dbConfig: Config, overrides?: Knex.Config, ) { - const knexConfig = buildPgDatabaseConfig(dbConfig, overrides); + const knexConfig = await buildPgDatabaseConfig(dbConfig, overrides); const database = knexFactory(knexConfig); const role = dbConfig.getOptionalString('role'); @@ -64,11 +64,11 @@ export function createPgDatabaseClient( * @param dbConfig - The database config * @param overrides - Additional options to merge with the config */ -export function buildPgDatabaseConfig( +export async function buildPgDatabaseConfig( dbConfig: Config, overrides?: Knex.Config, ) { - return mergeDatabaseConfig( + const config = mergeDatabaseConfig( dbConfig.get(), { connection: getPgConnectionConfig(dbConfig, !!overrides), @@ -76,6 +76,45 @@ export function buildPgDatabaseConfig( }, overrides, ); + + if (config.connection?.type === 'cloudsql') { + if (config.client !== 'pg') { + throw new Error('Cloud SQL only supports the pg client'); + } + + if (!config.connection.instance) { + throw new Error('Missing instance connection name for Cloud SQL'); + } + + const { + Connector: CloudSqlConnector, + IpAddressTypes, + AuthTypes, + } = await import('@google-cloud/cloud-sql-connector'); + const connector = new CloudSqlConnector(); + const clientOpts = await connector.getOptions({ + instanceConnectionName: config.connection.instance, + ipType: IpAddressTypes.PUBLIC, + authType: AuthTypes.IAM, + }); + + const cloudsqlConfig = { + ...config, + client: 'pg', + connection: { + ...config.connection, + ...clientOpts, + }, + }; + + // Trim additional properties from the connection object passed to knex + delete cloudsqlConfig.connection.type; + delete cloudsqlConfig.connection.instance; + + return cloudsqlConfig; + } + + return config; } /** @@ -130,7 +169,7 @@ export async function ensurePgDatabaseExists( dbConfig: Config, ...databases: Array ) { - const admin = createPgDatabaseClient(dbConfig, { + const admin = await createPgDatabaseClient(dbConfig, { connection: { database: 'postgres', }, @@ -186,7 +225,7 @@ export async function ensurePgSchemaExists( dbConfig: Config, ...schemas: Array ): Promise { - const admin = createPgDatabaseClient(dbConfig); + const admin = await createPgDatabaseClient(dbConfig); const role = dbConfig.getOptionalString('role'); try { @@ -219,7 +258,7 @@ export async function dropPgDatabase( dbConfig: Config, ...databases: Array ) { - const admin = createPgDatabaseClient(dbConfig); + const admin = await createPgDatabaseClient(dbConfig); try { await Promise.all( databases.map(async database => { diff --git a/yarn.lock b/yarn.lock index 11c894ed4c..3561e7e0d0 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3599,6 +3599,7 @@ __metadata: "@backstage/plugin-events-node": "workspace:^" "@backstage/plugin-permission-node": "workspace:^" "@backstage/types": "workspace:^" + "@google-cloud/cloud-sql-connector": ^1.4.0 "@google-cloud/storage": ^7.0.0 "@keyv/memcache": ^2.0.1 "@keyv/redis": ^4.0.1 @@ -3662,6 +3663,11 @@ __metadata: yauzl: ^3.0.0 yn: ^4.0.0 zod: ^3.22.4 + peerDependencies: + "@google-cloud/cloud-sql-connector": ^1.4.0 + peerDependenciesMeta: + "@google-cloud/cloud-sql-connector": + optional: true languageName: unknown linkType: soft @@ -10337,6 +10343,18 @@ __metadata: languageName: node linkType: hard +"@google-cloud/cloud-sql-connector@npm:^1.4.0": + version: 1.4.0 + resolution: "@google-cloud/cloud-sql-connector@npm:1.4.0" + dependencies: + "@googleapis/sqladmin": ^24.0.0 + gaxios: ^6.1.1 + google-auth-library: ^9.2.0 + p-throttle: ^5.1.0 + checksum: c4ac2cc8cc24cc5966f146fe5686493fdb0bb4fc43a27ec086bd116f5ce475ca88f5e5e5fb79d17e6c31fb1a752d2f7e94433db17eea787696556c45415a31a1 + languageName: node + linkType: hard + "@google-cloud/container@npm:^5.0.0": version: 5.19.0 resolution: "@google-cloud/container@npm:5.19.0" @@ -10406,6 +10424,15 @@ __metadata: languageName: node linkType: hard +"@googleapis/sqladmin@npm:^24.0.0": + version: 24.0.0 + resolution: "@googleapis/sqladmin@npm:24.0.0" + dependencies: + googleapis-common: ^7.0.0 + checksum: b1c93d18b800ecd5af79dc6728e6b030f6918a1c5e2094d2dd7b460e3b2033cb9ce4f4ffe5b6155ecc2f699b8e4bdf0a94999786671b8fa05e1d8f3ce90cf466 + languageName: node + linkType: hard + "@graphiql/react@npm:^0.20.3": version: 0.20.3 resolution: "@graphiql/react@npm:0.20.3" @@ -30109,15 +30136,16 @@ __metadata: languageName: node linkType: hard -"gaxios@npm:^6.0.0, gaxios@npm:^6.0.2, gaxios@npm:^6.1.1": - version: 6.3.0 - resolution: "gaxios@npm:6.3.0" +"gaxios@npm:^6.0.0, gaxios@npm:^6.0.2, gaxios@npm:^6.0.3, gaxios@npm:^6.1.1": + version: 6.7.1 + resolution: "gaxios@npm:6.7.1" dependencies: extend: ^3.0.2 https-proxy-agent: ^7.0.1 is-stream: ^2.0.0 node-fetch: ^2.6.9 - checksum: 4d4a8db32d833f8012435e2016cb0c919cac288e821bf81f877504e4284ef12b444cd903448e738c4031cd5219adf1e8d68e7df2b3dba774db9fde27f71109d4 + uuid: ^9.0.1 + checksum: ed5952655339918e0868c6f4e079d6e9e55b20a242ddb1be25076cdfb0dd1ca5a2cb233da7352baa972c19f898a78fa6ba67e7d848717c9ca9877c269b5b02f7 languageName: node linkType: hard @@ -30590,7 +30618,7 @@ __metadata: languageName: node linkType: hard -"google-auth-library@npm:^9.0.0, google-auth-library@npm:^9.3.0, google-auth-library@npm:^9.6.3": +"google-auth-library@npm:^9.0.0, google-auth-library@npm:^9.2.0, google-auth-library@npm:^9.3.0, google-auth-library@npm:^9.6.3, google-auth-library@npm:^9.7.0": version: 9.15.0 resolution: "google-auth-library@npm:9.15.0" dependencies: @@ -30631,6 +30659,20 @@ __metadata: languageName: node linkType: hard +"googleapis-common@npm:^7.0.0": + version: 7.2.0 + resolution: "googleapis-common@npm:7.2.0" + dependencies: + extend: ^3.0.2 + gaxios: ^6.0.3 + google-auth-library: ^9.7.0 + qs: ^6.7.0 + url-template: ^2.0.8 + uuid: ^9.0.0 + checksum: 58f520134c9d6f439ef81919471689f0278ef0ffdbd50c693a59282d95141be74df3a5614c25347c140fc44201e0ef998300389f4cbf51502f2351e67c758ab6 + languageName: node + linkType: hard + "gopd@npm:^1.0.1": version: 1.0.1 resolution: "gopd@npm:1.0.1" @@ -38301,6 +38343,13 @@ __metadata: languageName: node linkType: hard +"p-throttle@npm:^5.1.0": + version: 5.1.0 + resolution: "p-throttle@npm:5.1.0" + checksum: c412429cbb759a0772a083200a556e5866d4a7cb5383a9cc85880f47f07f4b1787fc33d2c824ab659fff0fc4a34a56e1d0712e88808f0c0ae0ffbe93b54f46c1 + languageName: node + linkType: hard + "p-timeout@npm:^3.2.0": version: 3.2.0 resolution: "p-timeout@npm:3.2.0" @@ -40263,7 +40312,7 @@ __metadata: languageName: node linkType: hard -"qs@npm:^6.10.1, qs@npm:^6.10.3, qs@npm:^6.11.0, qs@npm:^6.12.2, qs@npm:^6.9.4": +"qs@npm:^6.10.1, qs@npm:^6.10.3, qs@npm:^6.11.0, qs@npm:^6.12.2, qs@npm:^6.7.0, qs@npm:^6.9.4": version: 6.13.1 resolution: "qs@npm:6.13.1" dependencies: @@ -46108,6 +46157,13 @@ __metadata: languageName: node linkType: hard +"url-template@npm:^2.0.8": + version: 2.0.8 + resolution: "url-template@npm:2.0.8" + checksum: 4183fccd74e3591e4154134d4443dccecba9c455c15c7df774f1f1e3fa340fd9bffb903b5beec347196d15ce49c34edf6dec0634a95d170ad6e78c0467d6e13e + languageName: node + linkType: hard + "url-value-parser@npm:^2.0.0": version: 2.0.3 resolution: "url-value-parser@npm:2.0.3"