paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 43 Packages Projects Releases Wiki Activity

Merge pull request #27901 from backstage/blam/cloudsql

Browse Source 
Add support for `@google-cloud/cloud-sql-connector` database connection
This commit is contained in:
Ben Lambert
2024-12-02 11:29:57 +01:00
committed by GitHub
parent 01fa56bea8 ed5abc4340
commit c891a68539
6 changed files with 269 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.changeset/pink-pets-jump.md
+5
View File
@@ -0,0 +1,5 @@
---
'@backstage/backend-defaults': patch
---
Support `connection.type: cloudsql` in database client for usage with `@google-cloud/cloud-sql-connector` and `iam` auth
packages/backend-defaults/config.d.ts
Vendored
+34 -1
View File
@@ -384,6 +384,16 @@ export interface Config {
*/
connection:
| string
| {
/**
* The specific config for cloudsql connections
*/
type: 'cloudsql';
/**
* The instance connection name for the cloudsql instance, e.g. `project:region:instance`
*/
instance: string;
}
| {
/**
* Password that belongs to the client User
@@ -441,7 +451,30 @@ export interface Config {
* Database connection string or Knex object override
* @visibility secret
*/
connection?: string | object;
connection?:
| string
| {
/**
* The specific config for cloudsql connections
*/
type: 'cloudsql';
/**
* The instance connection name for the cloudsql instance, e.g. `project:region:instance`
*/
instance: string;
}
| {
/**
* Password that belongs to the client User
* @visibility secret
*/
password?: string;
/**
* Other connection settings
*/
[key: string]: unknown;
};
/**
* Whether to ensure the given database exists by creating it if it does not.
* Defaults to base config if unspecified.
packages/backend-defaults/package.json
+9
View File
@@ -185,11 +185,20 @@
"yn": "^4.0.0",
"zod": "^3.22.4"
},
"peerDependencies": {
"@google-cloud/cloud-sql-connector": "^1.4.0"
},
"peerDependenciesMeta": {
"@google-cloud/cloud-sql-connector": {
"optional": true
}
},
"devDependencies": {
"@aws-sdk/util-stream-node": "^3.350.0",
"@backstage/backend-plugin-api": "workspace:^",
"@backstage/backend-test-utils": "workspace:^",
"@backstage/cli": "workspace:^",
"@google-cloud/cloud-sql-connector": "^1.4.0",
"@types/archiver": "^6.0.0",
"@types/base64-stream": "^1.0.2",
"@types/concat-stream": "^2.0.0",
packages/backend-defaults/src/entrypoints/database/connectors/postgres.test.ts
+113 -26
View File
@@ -22,6 +22,8 @@ import {
parsePgConnectionString,
} from './postgres';
jest.mock('@google-cloud/cloud-sql-connector');
describe('postgres', () => {
const createMockConnection = () => ({
host: 'acme',
@@ -37,33 +39,35 @@ describe('postgres', () => {
new ConfigReader({ client: 'pg', connection });
describe('buildPgDatabaseConfig', () => {
it('builds a postgres config', () => {
it('builds a postgres config', async () => {
const mockConnection = createMockConnection();
expect(buildPgDatabaseConfig(createConfig(mockConnection))).toEqual({
client: 'pg',
connection: mockConnection,
useNullAsDefault: true,
});
});
it('builds a connection string config', () => {
const mockConnectionString = createMockConnectionString();
expect(buildPgDatabaseConfig(createConfig(mockConnectionString))).toEqual(
expect(await buildPgDatabaseConfig(createConfig(mockConnection))).toEqual(
{
client: 'pg',
connection: mockConnectionString,
connection: mockConnection,
useNullAsDefault: true,
},
);
});
it('overrides the database name', () => {
it('builds a connection string config', async () => {
const mockConnectionString = createMockConnectionString();
expect(
await buildPgDatabaseConfig(createConfig(mockConnectionString)),
).toEqual({
client: 'pg',
connection: mockConnectionString,
useNullAsDefault: true,
});
});
it('overrides the database name', async () => {
const mockConnection = createMockConnection();
expect(
buildPgDatabaseConfig(createConfig(mockConnection), {
await buildPgDatabaseConfig(createConfig(mockConnection), {
connection: { database: 'other_db' },
}),
).toEqual({
@@ -76,14 +80,14 @@ describe('postgres', () => {
});
});
it('overrides the schema name', () => {
it('overrides the schema name', async () => {
const mockConnection = {
...createMockConnection(),
schema: 'schemaName',
};
expect(
buildPgDatabaseConfig(createConfig(mockConnection), {
await buildPgDatabaseConfig(createConfig(mockConnection), {
searchPath: ['schemaName'],
}),
).toEqual({
@@ -94,11 +98,11 @@ describe('postgres', () => {
});
});
it('adds additional config settings', () => {
it('adds additional config settings', async () => {
const mockConnection = createMockConnection();
expect(
buildPgDatabaseConfig(createConfig(mockConnection), {
await buildPgDatabaseConfig(createConfig(mockConnection), {
connection: { database: 'other_db' },
pool: { min: 0, max: 7 },
debug: true,
@@ -115,12 +119,12 @@ describe('postgres', () => {
});
});
it('overrides the database from connection string', () => {
it('overrides the database from connection string', async () => {
const mockConnectionString = createMockConnectionString();
const mockConnection = createMockConnection();
expect(
buildPgDatabaseConfig(createConfig(mockConnectionString), {
await buildPgDatabaseConfig(createConfig(mockConnectionString), {
connection: { database: 'other_db' },
}),
).toEqual({
@@ -133,6 +137,89 @@ describe('postgres', () => {
useNullAsDefault: true,
});
});
it('uses the correct config when using cloudsql', async () => {
expect(
await buildPgDatabaseConfig(
new ConfigReader({
client: 'pg',
connection: {
type: 'cloudsql',
user: 'ben@gke.com',
instance: 'project:region:instance',
port: 5423,
},
}),
{ connection: { database: 'other_db' } },
),
).toEqual({
client: 'pg',
connection: {
user: 'ben@gke.com',
port: 5423,
database: 'other_db',
},
useNullAsDefault: true,
});
});
it('should throw with incorrect config', async () => {
await expect(
buildPgDatabaseConfig(
new ConfigReader({
client: 'pg',
connection: {
type: 'cloudsql',
},
}),
),
).rejects.toThrow(/Missing instance connection name for Cloud SQL/);
await expect(
buildPgDatabaseConfig(
new ConfigReader({
client: 'not-pg',
connection: {
type: 'cloudsql',
instance: 'asd:asd:asd',
},
}),
),
).rejects.toThrow(/Cloud SQL only supports the pg client/);
});
it('adds the settings from cloud-sql-connector', async () => {
const { Connector } = jest.requireMock(
'@google-cloud/cloud-sql-connector',
) as jest.Mocked<typeof import('@google-cloud/cloud-sql-connector')>;
const mockStream = (): any => {};
Connector.prototype.getOptions.mockResolvedValue({ stream: mockStream });
expect(
await buildPgDatabaseConfig(
new ConfigReader({
client: 'pg',
connection: {
type: 'cloudsql',
user: 'ben@gke.com',
instance: 'project:region:instance',
port: 5423,
},
}),
{ connection: { database: 'other_db' } },
),
).toEqual({
client: 'pg',
connection: {
user: 'ben@gke.com',
port: 5423,
stream: mockStream,
database: 'other_db',
},
useNullAsDefault: true,
});
});
});
describe('getPgConnectionConfig', () => {
@@ -174,9 +261,9 @@ describe('postgres', () => {
});
describe('createPgDatabaseClient', () => {
it('creates a postgres knex instance', () => {
it('creates a postgres knex instance', async () => {
expect(
createPgDatabaseClient(
await createPgDatabaseClient(
createConfig({
host: 'acme',
user: 'foo',
@@ -187,14 +274,14 @@ describe('postgres', () => {
).toBeTruthy();
});
it('attempts to read an ssl cert', () => {
expect(() =>
it('attempts to read an ssl cert', async () => {
await expect(() =>
createPgDatabaseClient(
createConfig(
'postgresql://postgres:pass@localhost:5432/dbname?sslrootcert=/path/to/file',
),
),
).toThrow(/no such file or directory/);
).rejects.toThrow(/no such file or directory/);
});
});
packages/backend-defaults/src/entrypoints/database/connectors/postgres.ts
+46 -7
View File
@@ -37,11 +37,11 @@ const ddlLimiter = limiterFactory(1);
* @param dbConfig - The database config
* @param overrides - Additional options to merge with the config
*/
export function createPgDatabaseClient(
export async function createPgDatabaseClient(
dbConfig: Config,
overrides?: Knex.Config,
) {
const knexConfig = buildPgDatabaseConfig(dbConfig, overrides);
const knexConfig = await buildPgDatabaseConfig(dbConfig, overrides);
const database = knexFactory(knexConfig);
const role = dbConfig.getOptionalString('role');
@@ -64,11 +64,11 @@ export function createPgDatabaseClient(
* @param dbConfig - The database config
* @param overrides - Additional options to merge with the config
*/
export function buildPgDatabaseConfig(
export async function buildPgDatabaseConfig(
dbConfig: Config,
overrides?: Knex.Config,
) {
return mergeDatabaseConfig(
const config = mergeDatabaseConfig(
dbConfig.get(),
{
connection: getPgConnectionConfig(dbConfig, !!overrides),
@@ -76,6 +76,45 @@ export function buildPgDatabaseConfig(
},
overrides,
);
if (config.connection?.type === 'cloudsql') {
if (config.client !== 'pg') {
throw new Error('Cloud SQL only supports the pg client');
}
if (!config.connection.instance) {
throw new Error('Missing instance connection name for Cloud SQL');
}
const {
Connector: CloudSqlConnector,
IpAddressTypes,
AuthTypes,
} = await import('@google-cloud/cloud-sql-connector');
const connector = new CloudSqlConnector();
const clientOpts = await connector.getOptions({
instanceConnectionName: config.connection.instance,
ipType: IpAddressTypes.PUBLIC,
authType: AuthTypes.IAM,
});
const cloudsqlConfig = {
...config,
client: 'pg',
connection: {
...config.connection,
...clientOpts,
},
};
// Trim additional properties from the connection object passed to knex
delete cloudsqlConfig.connection.type;
delete cloudsqlConfig.connection.instance;
return cloudsqlConfig;
}
return config;
}
/**
@@ -130,7 +169,7 @@ export async function ensurePgDatabaseExists(
dbConfig: Config,
...databases: Array<string>
) {
const admin = createPgDatabaseClient(dbConfig, {
const admin = await createPgDatabaseClient(dbConfig, {
connection: {
database: 'postgres',
},
@@ -186,7 +225,7 @@ export async function ensurePgSchemaExists(
dbConfig: Config,
...schemas: Array<string>
): Promise<void> {
const admin = createPgDatabaseClient(dbConfig);
const admin = await createPgDatabaseClient(dbConfig);
const role = dbConfig.getOptionalString('role');
try {
@@ -219,7 +258,7 @@ export async function dropPgDatabase(
dbConfig: Config,
...databases: Array<string>
) {
const admin = createPgDatabaseClient(dbConfig);
const admin = await createPgDatabaseClient(dbConfig);
try {
await Promise.all(
databases.map(async database => {
yarn.lock
+62 -6
View File
@@ -3599,6 +3599,7 @@ __metadata:
"@backstage/plugin-events-node": "workspace:^"
"@backstage/plugin-permission-node": "workspace:^"
"@backstage/types": "workspace:^"
"@google-cloud/cloud-sql-connector": ^1.4.0
"@google-cloud/storage": ^7.0.0
"@keyv/memcache": ^2.0.1
"@keyv/redis": ^4.0.1
@@ -3662,6 +3663,11 @@ __metadata:
yauzl: ^3.0.0
yn: ^4.0.0
zod: ^3.22.4
peerDependencies:
"@google-cloud/cloud-sql-connector": ^1.4.0
peerDependenciesMeta:
"@google-cloud/cloud-sql-connector":
optional: true
languageName: unknown
linkType: soft
@@ -10337,6 +10343,18 @@ __metadata:
languageName: node
linkType: hard
"@google-cloud/cloud-sql-connector@npm:^1.4.0":
version: 1.4.0
resolution: "@google-cloud/cloud-sql-connector@npm:1.4.0"
dependencies:
"@googleapis/sqladmin": ^24.0.0
gaxios: ^6.1.1
google-auth-library: ^9.2.0
p-throttle: ^5.1.0
checksum: c4ac2cc8cc24cc5966f146fe5686493fdb0bb4fc43a27ec086bd116f5ce475ca88f5e5e5fb79d17e6c31fb1a752d2f7e94433db17eea787696556c45415a31a1
languageName: node
linkType: hard
"@google-cloud/container@npm:^5.0.0":
version: 5.19.0
resolution: "@google-cloud/container@npm:5.19.0"
@@ -10406,6 +10424,15 @@ __metadata:
languageName: node
linkType: hard
"@googleapis/sqladmin@npm:^24.0.0":
version: 24.0.0
resolution: "@googleapis/sqladmin@npm:24.0.0"
dependencies:
googleapis-common: ^7.0.0
checksum: b1c93d18b800ecd5af79dc6728e6b030f6918a1c5e2094d2dd7b460e3b2033cb9ce4f4ffe5b6155ecc2f699b8e4bdf0a94999786671b8fa05e1d8f3ce90cf466
languageName: node
linkType: hard
"@graphiql/react@npm:^0.20.3":
version: 0.20.3
resolution: "@graphiql/react@npm:0.20.3"
@@ -30109,15 +30136,16 @@ __metadata:
languageName: node
linkType: hard
"gaxios@npm:^6.0.0, gaxios@npm:^6.0.2, gaxios@npm:^6.1.1":
version: 6.3.0
resolution: "gaxios@npm:6.3.0"
"gaxios@npm:^6.0.0, gaxios@npm:^6.0.2, gaxios@npm:^6.0.3, gaxios@npm:^6.1.1":
version: 6.7.1
resolution: "gaxios@npm:6.7.1"
dependencies:
extend: ^3.0.2
https-proxy-agent: ^7.0.1
is-stream: ^2.0.0
node-fetch: ^2.6.9
checksum: 4d4a8db32d833f8012435e2016cb0c919cac288e821bf81f877504e4284ef12b444cd903448e738c4031cd5219adf1e8d68e7df2b3dba774db9fde27f71109d4
uuid: ^9.0.1
checksum: ed5952655339918e0868c6f4e079d6e9e55b20a242ddb1be25076cdfb0dd1ca5a2cb233da7352baa972c19f898a78fa6ba67e7d848717c9ca9877c269b5b02f7
languageName: node
linkType: hard
@@ -30590,7 +30618,7 @@ __metadata:
languageName: node
linkType: hard
"google-auth-library@npm:^9.0.0, google-auth-library@npm:^9.3.0, google-auth-library@npm:^9.6.3":
"google-auth-library@npm:^9.0.0, google-auth-library@npm:^9.2.0, google-auth-library@npm:^9.3.0, google-auth-library@npm:^9.6.3, google-auth-library@npm:^9.7.0":
version: 9.15.0
resolution: "google-auth-library@npm:9.15.0"
dependencies:
@@ -30631,6 +30659,20 @@ __metadata:
languageName: node
linkType: hard
"googleapis-common@npm:^7.0.0":
version: 7.2.0
resolution: "googleapis-common@npm:7.2.0"
dependencies:
extend: ^3.0.2
gaxios: ^6.0.3
google-auth-library: ^9.7.0
qs: ^6.7.0
url-template: ^2.0.8
uuid: ^9.0.0
checksum: 58f520134c9d6f439ef81919471689f0278ef0ffdbd50c693a59282d95141be74df3a5614c25347c140fc44201e0ef998300389f4cbf51502f2351e67c758ab6
languageName: node
linkType: hard
"gopd@npm:^1.0.1":
version: 1.0.1
resolution: "gopd@npm:1.0.1"
@@ -38301,6 +38343,13 @@ __metadata:
languageName: node
linkType: hard
"p-throttle@npm:^5.1.0":
version: 5.1.0
resolution: "p-throttle@npm:5.1.0"
checksum: c412429cbb759a0772a083200a556e5866d4a7cb5383a9cc85880f47f07f4b1787fc33d2c824ab659fff0fc4a34a56e1d0712e88808f0c0ae0ffbe93b54f46c1
languageName: node
linkType: hard
"p-timeout@npm:^3.2.0":
version: 3.2.0
resolution: "p-timeout@npm:3.2.0"
@@ -40263,7 +40312,7 @@ __metadata:
languageName: node
linkType: hard
"qs@npm:^6.10.1, qs@npm:^6.10.3, qs@npm:^6.11.0, qs@npm:^6.12.2, qs@npm:^6.9.4":
"qs@npm:^6.10.1, qs@npm:^6.10.3, qs@npm:^6.11.0, qs@npm:^6.12.2, qs@npm:^6.7.0, qs@npm:^6.9.4":
version: 6.13.1
resolution: "qs@npm:6.13.1"
dependencies:
@@ -46108,6 +46157,13 @@ __metadata:
languageName: node
linkType: hard
"url-template@npm:^2.0.8":
version: 2.0.8
resolution: "url-template@npm:2.0.8"
checksum: 4183fccd74e3591e4154134d4443dccecba9c455c15c7df774f1f1e3fa340fd9bffb903b5beec347196d15ce49c34edf6dec0634a95d170ad6e78c0467d6e13e
languageName: node
linkType: hard
"url-value-parser@npm:^2.0.0":
version: 2.0.3
resolution: "url-value-parser@npm:2.0.3"