Merge pull request #19988 from backstage/rugvip/retry

backend-common: added DB creation retries and lowered pool limits

.changeset/sour-rules-press.md

@@ -0,0 +1,5 @@
+---
+'@backstage/backend-common': patch
+---
+
+Added retries for initial database creation, as well as set minimum connection pool size for the database creation client to 0 and lowered the connection acquisition timeout.

packages/backend-common/src/database/connectors/mysql.ts

@@ -151,13 +151,33 @@ export async function ensureMysqlDatabaseExists(
     connection: {
       database: null as unknown as string,
     },
+    pool: {
+      min: 0,
+      acquireTimeoutMillis: 10000,
+    },
   });
 
   try {
     const ensureDatabase = async (database: string) => {
       await admin.raw(`CREATE DATABASE IF NOT EXISTS ??`, [database]);
     };
-    await Promise.all(databases.map(ensureDatabase));
+    await Promise.all(
+      databases.map(async database => {
+        // For initial setup we use a smaller timeout but several retries. Given that this
+        // is a separate connection pool we should never really run into issues with connection
+        // acquisition timeouts, but we do anyway. This might be a bug in knex or some other dependency.
+        let lastErr: Error | undefined = undefined;
+        for (let i = 0; i < 3; i++) {
+          try {
+            return await ensureDatabase(database);
+          } catch (err) {
+            lastErr = err;
+          }
+          await new Promise(resolve => setTimeout(resolve, 100));
+        }
+        throw lastErr;
+      }),
+    );
   } finally {
     await admin.destroy();
   }

packages/backend-common/src/database/connectors/postgres.ts

@@ -126,6 +126,10 @@ export async function ensurePgDatabaseExists(
     connection: {
       database: 'postgres',
     },
+    pool: {
+      min: 0,
+      acquireTimeoutMillis: 10000,
+    },
   });
 
   try {
@@ -142,7 +146,23 @@ export async function ensurePgDatabaseExists(
       await admin.raw(`CREATE DATABASE ??`, [database]);
     };
 
-    await Promise.all(databases.map(ensureDatabase));
+    await Promise.all(
+      databases.map(async database => {
+        // For initial setup we use a smaller timeout but several retries. Given that this
+        // is a separate connection pool we should never really run into issues with connection
+        // acquisition timeouts, but we do anyway. This might be a bug in knex or some other dependency.
+        let lastErr: Error | undefined = undefined;
+        for (let i = 0; i < 3; i++) {
+          try {
+            return await ensureDatabase(database);
+          } catch (err) {
+            lastErr = err;
+          }
+          await new Promise(resolve => setTimeout(resolve, 100));
+        }
+        throw lastErr;
+      }),
+    );
   } finally {
     await admin.destroy();
   }