Merge pull request #10948 from backstage/jhaals/simplify-create-app

Remove database choice from `create-app` command

.changeset/smart-ghosts-search.md

@@ -0,0 +1,28 @@
+---
+'@backstage/create-app': patch
+---
+
+Removed the database choice from the `create-app` command.
+
+This reduces the step from development to production by always installing the dependencies and templating the production configuration in `app-config.production.yaml`.
+
+Added `app-config.local.yaml` to allow for local configuration overrides.
+To replicate this behavior in an existing installation simply `touch app-config.local.yaml` in the project root and apply your local configuration.
+
+`better-sqlite3` has been moved to devDependencies, for existing installations using postgres in production and SQLite in development it's recommended to move SQLite into the devDependencies section to avoid unnecessary dependencies during builds.
+
+in `packages/backend/package.json`
+
+```diff
+  "dependencies": {
+    ...
+    "pg": "^8.3.0",
+-   "better-sqlite3": "^7.5.0",
+    "winston": "^3.2.1"
+  },
+  "devDependencies": {
+    ...
+    "@types/luxon": "^2.0.4",
++   "better-sqlite3": "^7.5.0"
+  }
+```

.github/workflows/verify_e2e-linux.yml

@@ -78,3 +78,7 @@ jobs:
           yarn e2e-test run
         env:
           BACKSTAGE_TEST_DISABLE_DOCKER: 1
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_HOST: localhost
+          POSTGRES_PORT: ${{ job.services.postgres.ports[5432] }}

.gitignore

@@ -130,6 +130,7 @@ site
 
 # Local configuration files
 *.local.yaml
+!packages/create-app/templates/default-app/app-config.local.yaml
 
 # Sensitive credentials
 *-credentials.yaml

packages/create-app/src/createApp.ts

@@ -50,15 +50,7 @@ export default async (opts: OptionValues): Promise<void> => {
         return true;
       },
     },
-    {
-      type: 'list',
-      name: 'dbType',
-      message: chalk.blue('Select database for the backend [required]'),
-      choices: ['SQLite', 'PostgreSQL'],
-    },
   ]);
-  answers.dbTypePG = answers.dbType === 'PostgreSQL';
-  answers.dbTypeSqlite = answers.dbType === 'SQLite';
 
   const templateDir = paths.resolveOwn('templates/default-app');
   const tempDir = resolvePath(os.tmpdir(), answers.name);

packages/create-app/templates/default-app/app-config.local.yaml

@@ -0,0 +1 @@
+# Backstage override configuration for your local development environment

packages/create-app/templates/default-app/app-config.production.yaml

@@ -17,3 +17,18 @@ backend:
     # backend will bind on the interface that corresponds to the backend.baseUrl
     # hostname.
     host: 0.0.0.0
+
+  # config options: https://node-postgres.com/api/client
+  database:
+    client: pg
+    connection:
+      host: ${POSTGRES_HOST}
+      port: ${POSTGRES_PORT}
+      user: ${POSTGRES_USER}
+      password: ${POSTGRES_PASSWORD}
+      # https://node-postgres.com/features/ssl
+      # you can set the sslmode configuration option via the `PGSSLMODE` environment variable
+      # see https://www.postgresql.org/docs/current/libpq-ssl.html Table 33.1. SSL Mode Descriptions (e.g. require)
+      # ssl:
+      #   ca: # if you have a CA file and want to verify it you can uncomment this section
+      #     $file: <file-path>/ca/server.crt

packages/create-app/templates/default-app/app-config.yaml.hbs

@@ -26,27 +26,11 @@ backend:
     origin: http://localhost:3000
     methods: [GET, POST, PUT, DELETE]
     credentials: true
-  {{#if dbTypeSqlite}}
+  # This is for local developement only, it is not recommended to use this in production
+  # The production database configuration is stored in app-config.production.yaml
   database:
     client: better-sqlite3
     connection: ':memory:'
-  {{/if}}
-  {{#if dbTypePG}}
-  # config options: https://node-postgres.com/api/client
-  database:
-    client: pg
-    connection:
-      host: ${POSTGRES_HOST}
-      port: ${POSTGRES_PORT}
-      user: ${POSTGRES_USER}
-      password: ${POSTGRES_PASSWORD}
-      # https://node-postgres.com/features/ssl
-      # you can set the sslmode configuration option via the `PGSSLMODE` environment variable
-      # see https://www.postgresql.org/docs/current/libpq-ssl.html Table 33.1. SSL Mode Descriptions (e.g. require)
-      # ssl:
-      #   ca: # if you have a CA file and want to verify it you can uncomment this section
-      #     $file: <file-path>/ca/server.crt
-  {{/if}}
   cache:
     store: memory
   # workingDirectory: /tmp # Use this to configure a working directory for the scaffolder, defaults to the OS temp-dir

packages/create-app/templates/default-app/packages/backend/package.json.hbs

@@ -30,29 +30,23 @@
     "@backstage/plugin-proxy-backend": "^{{version '@backstage/plugin-proxy-backend'}}",
     "@backstage/plugin-scaffolder-backend": "^{{version '@backstage/plugin-scaffolder-backend'}}",
     "@backstage/plugin-search-backend": "^{{version '@backstage/plugin-search-backend'}}",
-    {{#if dbTypePG}}
     "@backstage/plugin-search-backend-module-pg": "^{{version '@backstage/plugin-search-backend-module-pg'}}",
-    {{/if}}
     "@backstage/plugin-search-backend-node": "^{{version '@backstage/plugin-search-backend-node'}}",
     "@backstage/plugin-techdocs-backend": "^{{version '@backstage/plugin-techdocs-backend'}}",
     "dockerode": "^3.3.1",
     "express": "^4.17.1",
     "express-promise-router": "^4.1.0",
     "luxon": "^2.0.2",
-    {{#if dbTypePG}}
     "pg": "^8.3.0",
-    {{/if}}
-    {{#if dbTypeSqlite}}
-    "better-sqlite3": "^7.5.0",
-    {{/if}}
     "winston": "^3.2.1"
   },
   "devDependencies": {
     "@backstage/cli": "^{{version '@backstage/cli'}}",
     "@types/dockerode": "^3.3.0",
-    "@types/express": "^4.17.6",
     "@types/express-serve-static-core": "^4.17.5",
-    "@types/luxon": "^2.0.4"
+    "@types/express": "^4.17.6",
+    "@types/luxon": "^2.0.4",
+    "better-sqlite3": "^7.5.0"
   },
   "files": [
     "dist"

packages/create-app/templates/default-app/packages/backend/src/plugins/search.ts.hbs

@@ -4,9 +4,6 @@ import {
   IndexBuilder,
   LunrSearchEngine,
 } from '@backstage/plugin-search-backend-node';
-{{#if dbTypePG}}
-import { PgSearchEngine } from '@backstage/plugin-search-backend-module-pg';
-{{/if}}
 import { PluginEnvironment } from '../types';
 import { DefaultCatalogCollatorFactory } from '@backstage/plugin-catalog-backend';
 import { DefaultTechDocsCollatorFactory } from '@backstage/plugin-techdocs-backend';
@@ -17,16 +14,9 @@ export default async function createPlugin(
   env: PluginEnvironment,
 ): Promise<Router> {
   // Initialize a connection to a search engine.
-  {{#if dbTypeSqlite}}
   const searchEngine = new LunrSearchEngine({
     logger: env.logger,
   });
-  {{/if}}
-  {{#if dbTypePG}}
-  const searchEngine = (await PgSearchEngine.supported(env.database))
-    ? await PgSearchEngine.from({ database: env.database })
-    : new LunrSearchEngine({ logger: env.logger });
-  {{/if}}
   const indexBuilder = new IndexBuilder({
     logger: env.logger,
     searchEngine,

packages/e2e-test/src/commands/run.ts

@@ -21,6 +21,7 @@ import handlebars from 'handlebars';
 import killTree from 'tree-kill';
 import { resolve as resolvePath, join as joinPath } from 'path';
 import puppeteer from 'puppeteer';
+import path from 'path';
 
 import {
   spawnPiped,
@@ -49,9 +50,8 @@ export async function run() {
   print('Building dist workspace');
   const workspaceDir = await buildDistWorkspace('workspace', rootDir);
 
-  const isPostgres = Boolean(process.env.POSTGRES_USER);
   print('Creating a Backstage App');
-  const appDir = await createApp('test-app', isPostgres, workspaceDir, rootDir);
+  const appDir = await createApp('test-app', workspaceDir, rootDir);
 
   print('Creating a Backstage Plugin');
   const pluginName = await createPlugin('test-plugin', appDir);
@@ -62,8 +62,21 @@ export async function run() {
   print('Starting the app');
   await testAppServe(pluginName, appDir);
 
-  print('Testing the backend startup');
-  await testBackendStart(appDir, isPostgres);
+  if (Boolean(process.env.POSTGRES_USER)) {
+    print('Testing the PostgreSQL backend startup');
+    await preCleanPostgres();
+    const appConfig = path.resolve(appDir, 'app-config.yaml');
+    const productionConfig = path.resolve(appDir, 'app-config.production.yaml');
+    await testBackendStart(
+      appDir,
+      '--config',
+      appConfig,
+      '--config',
+      productionConfig,
+    );
+  }
+  print('Testing the SQLite backend startup');
+  await testBackendStart(appDir);
 
   if (process.env.CI) {
     // Cleanup actually takes significant time, so skip it in CI since the
@@ -100,8 +113,8 @@ async function buildDistWorkspace(workspaceName: string, rootDir: string) {
   }
 
   for (const pkgJsonPath of templatePackagePaths) {
-    const path = paths.resolveOwnRoot(pkgJsonPath);
-    const pkgTemplate = await fs.readFile(path, 'utf8');
+    const jsonPath = paths.resolveOwnRoot(pkgJsonPath);
+    const pkgTemplate = await fs.readFile(jsonPath, 'utf8');
     const pkg = JSON.parse(
       handlebars.compile(pkgTemplate)(
         {
@@ -191,7 +204,6 @@ async function pinYarnVersion(dir: string) {
  */
 async function createApp(
   appName: string,
-  isPostgres: boolean,
   workspaceDir: string,
   rootDir: string,
 ) {
@@ -215,14 +227,6 @@ async function createApp(
     await waitFor(() => stdout.includes('Enter a name for the app'));
     child.stdin?.write(`${appName}\n`);
 
-    await waitFor(() => stdout.includes('Select database for the backend'));
-
-    if (isPostgres) {
-      // Simulate down arrow press
-      child.stdin?.write(`\u001B\u005B\u0042`);
-    }
-    child.stdin?.write(`\n`);
-
     print('Waiting for app create script to be done');
     await waitForExit(child);
 
@@ -404,33 +408,34 @@ async function dropDB(database: string) {
   };
 
   try {
-    await pgtools.dropdb({ config }, database);
+    await pgtools.dropdb(config, database);
   } catch (_) {
     /* do nothing*/
   }
 }
 
+/** Clean remnants from prior e2e runs */
+async function preCleanPostgres() {
+  print('Dropping old DBs');
+  await Promise.all(
+    [
+      'catalog',
+      'scaffolder',
+      'auth',
+      'identity',
+      'proxy',
+      'techdocs',
+      'search',
+    ].map(name => dropDB(`backstage_plugin_${name}`)),
+  );
+  print('Created DBs');
+}
+
 /**
  * Start serving the newly created backend and make sure that all db migrations works correctly
  */
-async function testBackendStart(appDir: string, isPostgres: boolean) {
-  if (isPostgres) {
-    print('Dropping old DBs');
-    await Promise.all(
-      [
-        'catalog',
-        'scaffolder',
-        'auth',
-        'identity',
-        'proxy',
-        'techdocs',
-        'search',
-      ].map(name => dropDB(`backstage_plugin_${name}`)),
-    );
-    print('Created DBs');
-  }
-
-  const child = spawnPiped(['yarn', 'workspace', 'backend', 'start'], {
+async function testBackendStart(appDir: string, ...args: string[]) {
+  const child = spawnPiped(['yarn', 'workspace', 'backend', 'start', ...args], {
     cwd: appDir,
     env: {
       ...process.env,