diff --git a/packages/backend-defaults/package.json b/packages/backend-defaults/package.json index 689688630e..4612cac52b 100644 --- a/packages/backend-defaults/package.json +++ b/packages/backend-defaults/package.json @@ -185,6 +185,14 @@ "yn": "^4.0.0", "zod": "^3.22.4" }, + "peerDependencies": { + "@google-cloud/cloud-sql-connector": "^1.4.0" + }, + "peerDependenciesMeta": { + "@google-cloud/cloud-sql-connector": { + "optional": true + } + }, "devDependencies": { "@aws-sdk/util-stream-node": "^3.350.0", "@backstage/backend-plugin-api": "workspace:^", diff --git a/packages/backend-defaults/src/entrypoints/database/DatabaseManager.ts b/packages/backend-defaults/src/entrypoints/database/DatabaseManager.ts index 75e7d35eee..4ea66de2cc 100644 --- a/packages/backend-defaults/src/entrypoints/database/DatabaseManager.ts +++ b/packages/backend-defaults/src/entrypoints/database/DatabaseManager.ts @@ -252,6 +252,7 @@ export class DatabaseManager { databaseConfig, { pg: new PgConnector(databaseConfig, prefix), + 'pg+cloudsql': new PgConnector(databaseConfig, prefix), sqlite3: new Sqlite3Connector(databaseConfig), 'better-sqlite3': new Sqlite3Connector(databaseConfig), mysql: new MysqlConnector(databaseConfig, prefix), diff --git a/packages/backend-defaults/src/entrypoints/database/connectors/postgres.test.ts b/packages/backend-defaults/src/entrypoints/database/connectors/postgres.test.ts index 1e07984967..55568bc816 100644 --- a/packages/backend-defaults/src/entrypoints/database/connectors/postgres.test.ts +++ b/packages/backend-defaults/src/entrypoints/database/connectors/postgres.test.ts @@ -22,6 +22,8 @@ import { parsePgConnectionString, } from './postgres'; +jest.mock('@google-cloud/cloud-sql-connector'); + describe('postgres', () => { const createMockConnection = () => ({ host: 'acme', @@ -37,33 +39,35 @@ describe('postgres', () => { new ConfigReader({ client: 'pg', connection }); describe('buildPgDatabaseConfig', () => { - it('builds a postgres config', () => { + it('builds a postgres config', async () => { const mockConnection = createMockConnection(); - expect(buildPgDatabaseConfig(createConfig(mockConnection))).toEqual({ - client: 'pg', - connection: mockConnection, - useNullAsDefault: true, - }); - }); - - it('builds a connection string config', () => { - const mockConnectionString = createMockConnectionString(); - - expect(buildPgDatabaseConfig(createConfig(mockConnectionString))).toEqual( + expect(await buildPgDatabaseConfig(createConfig(mockConnection))).toEqual( { client: 'pg', - connection: mockConnectionString, + connection: mockConnection, useNullAsDefault: true, }, ); }); - it('overrides the database name', () => { + it('builds a connection string config', async () => { + const mockConnectionString = createMockConnectionString(); + + expect( + await buildPgDatabaseConfig(createConfig(mockConnectionString)), + ).toEqual({ + client: 'pg', + connection: mockConnectionString, + useNullAsDefault: true, + }); + }); + + it('overrides the database name', async () => { const mockConnection = createMockConnection(); expect( - buildPgDatabaseConfig(createConfig(mockConnection), { + await buildPgDatabaseConfig(createConfig(mockConnection), { connection: { database: 'other_db' }, }), ).toEqual({ @@ -76,14 +80,14 @@ describe('postgres', () => { }); }); - it('overrides the schema name', () => { + it('overrides the schema name', async () => { const mockConnection = { ...createMockConnection(), schema: 'schemaName', }; expect( - buildPgDatabaseConfig(createConfig(mockConnection), { + await buildPgDatabaseConfig(createConfig(mockConnection), { searchPath: ['schemaName'], }), ).toEqual({ @@ -94,11 +98,11 @@ describe('postgres', () => { }); }); - it('adds additional config settings', () => { + it('adds additional config settings', async () => { const mockConnection = createMockConnection(); expect( - buildPgDatabaseConfig(createConfig(mockConnection), { + await buildPgDatabaseConfig(createConfig(mockConnection), { connection: { database: 'other_db' }, pool: { min: 0, max: 7 }, debug: true, @@ -115,12 +119,12 @@ describe('postgres', () => { }); }); - it('overrides the database from connection string', () => { + it('overrides the database from connection string', async () => { const mockConnectionString = createMockConnectionString(); const mockConnection = createMockConnection(); expect( - buildPgDatabaseConfig(createConfig(mockConnectionString), { + await buildPgDatabaseConfig(createConfig(mockConnectionString), { connection: { database: 'other_db' }, }), ).toEqual({ @@ -133,6 +137,64 @@ describe('postgres', () => { useNullAsDefault: true, }); }); + + it('uses the correct config when using pg+google-cloud-sql', async () => { + const mockConnectionString = createMockConnectionString(); + const mockConnection = createMockConnection(); + + expect( + await buildPgDatabaseConfig( + new ConfigReader({ + client: 'pg+google-cloud-sql', + connection: mockConnectionString, + instanceConnectionName: 'project:region:instance', + }), + { connection: { database: 'other_db' } }, + ), + ).toEqual({ + client: 'pg', + connection: { + ...mockConnection, + port: '5432', + database: 'other_db', + }, + instanceConnectionName: 'project:region:instance', + useNullAsDefault: true, + }); + }); + + it('adds the settings from cloud-sql-connector', async () => { + const { Connector } = jest.requireMock( + '@google-cloud/cloud-sql-connector', + ) as jest.Mocked; + + const mockStream = (): any => {}; + Connector.prototype.getOptions.mockResolvedValue({ stream: mockStream }); + + const mockConnectionString = createMockConnectionString(); + const mockConnection = createMockConnection(); + + expect( + await buildPgDatabaseConfig( + new ConfigReader({ + client: 'pg+google-cloud-sql', + connection: mockConnectionString, + instanceConnectionName: 'project:region:instance', + }), + { connection: { database: 'other_db' } }, + ), + ).toEqual({ + client: 'pg', + connection: { + ...mockConnection, + port: '5432', + database: 'other_db', + stream: mockStream, + }, + instanceConnectionName: 'project:region:instance', + useNullAsDefault: true, + }); + }); }); describe('getPgConnectionConfig', () => { @@ -174,9 +236,9 @@ describe('postgres', () => { }); describe('createPgDatabaseClient', () => { - it('creates a postgres knex instance', () => { + it('creates a postgres knex instance', async () => { expect( - createPgDatabaseClient( + await createPgDatabaseClient( createConfig({ host: 'acme', user: 'foo', @@ -187,14 +249,14 @@ describe('postgres', () => { ).toBeTruthy(); }); - it('attempts to read an ssl cert', () => { - expect(() => + it('attempts to read an ssl cert', async () => { + await expect(() => createPgDatabaseClient( createConfig( 'postgresql://postgres:pass@localhost:5432/dbname?sslrootcert=/path/to/file', ), ), - ).toThrow(/no such file or directory/); + ).rejects.toThrow(/no such file or directory/); }); }); diff --git a/packages/backend-defaults/src/entrypoints/database/connectors/postgres.ts b/packages/backend-defaults/src/entrypoints/database/connectors/postgres.ts index b5cfffd1f4..c81d352b74 100644 --- a/packages/backend-defaults/src/entrypoints/database/connectors/postgres.ts +++ b/packages/backend-defaults/src/entrypoints/database/connectors/postgres.ts @@ -37,11 +37,11 @@ const ddlLimiter = limiterFactory(1); * @param dbConfig - The database config * @param overrides - Additional options to merge with the config */ -export function createPgDatabaseClient( +export async function createPgDatabaseClient( dbConfig: Config, overrides?: Knex.Config, ) { - const knexConfig = buildPgDatabaseConfig(dbConfig, overrides); + const knexConfig = await buildPgDatabaseConfig(dbConfig, overrides); const database = knexFactory(knexConfig); const role = dbConfig.getOptionalString('role'); @@ -64,11 +64,11 @@ export function createPgDatabaseClient( * @param dbConfig - The database config * @param overrides - Additional options to merge with the config */ -export function buildPgDatabaseConfig( +export async function buildPgDatabaseConfig( dbConfig: Config, overrides?: Knex.Config, ) { - return mergeDatabaseConfig( + const config = mergeDatabaseConfig( dbConfig.get(), { connection: getPgConnectionConfig(dbConfig, !!overrides), @@ -76,6 +76,33 @@ export function buildPgDatabaseConfig( }, overrides, ); + + if (config.client === 'pg+google-cloud-sql') { + const { + Connector: CloudSqlConnector, + IpAddressTypes, + AuthTypes, + } = await import('@google-cloud/cloud-sql-connector'); + // override the config to be pg for backwards compat with other code + config.client = 'pg'; + + const connector = new CloudSqlConnector(); + const clientOpts = await connector.getOptions({ + instanceConnectionName: dbConfig.getString('instanceConnectionName'), + ipType: IpAddressTypes.PUBLIC, + authType: AuthTypes.IAM, + }); + + return { + ...config, + connection: { + ...config.connection, + ...clientOpts, + }, + }; + } + + return config; } /** @@ -130,7 +157,7 @@ export async function ensurePgDatabaseExists( dbConfig: Config, ...databases: Array ) { - const admin = createPgDatabaseClient(dbConfig, { + const admin = await createPgDatabaseClient(dbConfig, { connection: { database: 'postgres', }, @@ -186,7 +213,7 @@ export async function ensurePgSchemaExists( dbConfig: Config, ...schemas: Array ): Promise { - const admin = createPgDatabaseClient(dbConfig); + const admin = await createPgDatabaseClient(dbConfig); const role = dbConfig.getOptionalString('role'); try { @@ -219,7 +246,7 @@ export async function dropPgDatabase( dbConfig: Config, ...databases: Array ) { - const admin = createPgDatabaseClient(dbConfig); + const admin = await createPgDatabaseClient(dbConfig); try { await Promise.all( databases.map(async database => { diff --git a/yarn.lock b/yarn.lock index 3075fcd983..915e711688 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3662,6 +3662,11 @@ __metadata: yauzl: ^3.0.0 yn: ^4.0.0 zod: ^3.22.4 + peerDependencies: + "@google-cloud/cloud-sql-connector": ^1.4.0 + peerDependenciesMeta: + "@google-cloud/cloud-sql-connector": + optional: true languageName: unknown linkType: soft @@ -30112,14 +30117,15 @@ __metadata: linkType: hard "gaxios@npm:^6.0.0, gaxios@npm:^6.0.2, gaxios@npm:^6.1.1": - version: 6.3.0 - resolution: "gaxios@npm:6.3.0" + version: 6.7.1 + resolution: "gaxios@npm:6.7.1" dependencies: extend: ^3.0.2 https-proxy-agent: ^7.0.1 is-stream: ^2.0.0 node-fetch: ^2.6.9 - checksum: 4d4a8db32d833f8012435e2016cb0c919cac288e821bf81f877504e4284ef12b444cd903448e738c4031cd5219adf1e8d68e7df2b3dba774db9fde27f71109d4 + uuid: ^9.0.1 + checksum: ed5952655339918e0868c6f4e079d6e9e55b20a242ddb1be25076cdfb0dd1ca5a2cb233da7352baa972c19f898a78fa6ba67e7d848717c9ca9877c269b5b02f7 languageName: node linkType: hard