From 00fa0dada002ee96b035600322ac0d0e9f0bf61d Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Wed, 20 Apr 2022 15:08:20 +0200 Subject: [PATCH 01/10] Remove database choice from `create-app` command Signed-off-by: Johan Haals --- .changeset/smart-ghosts-search.md | 9 +++++++++ .gitignore | 1 + packages/create-app/src/createApp.ts | 8 -------- .../default-app/app-config.local.yaml | 1 + .../default-app/app-config.production.yaml | 15 ++++++++++++++ .../templates/default-app/app-config.yaml.hbs | 20 ++----------------- .../packages/backend/package.json.hbs | 8 +------- 7 files changed, 29 insertions(+), 33 deletions(-) create mode 100644 .changeset/smart-ghosts-search.md create mode 100644 packages/create-app/templates/default-app/app-config.local.yaml diff --git a/.changeset/smart-ghosts-search.md b/.changeset/smart-ghosts-search.md new file mode 100644 index 0000000000..5d8b3676cb --- /dev/null +++ b/.changeset/smart-ghosts-search.md @@ -0,0 +1,9 @@ +--- +'@backstage/create-app': patch +--- + +Removed the database choice from the `create-app` command. + +This reduces the step from development to production by always installing the dependencies and templating the production configuration in `app-config.production.yaml`. + +Added `app-config.local.yaml` to allow for local configuration overrides. diff --git a/.gitignore b/.gitignore index 60d0d6da3b..a54955ac73 100644 --- a/.gitignore +++ b/.gitignore @@ -130,6 +130,7 @@ site # Local configuration files *.local.yaml +!packages/create-app/templates/default-app/app-config.local.yaml # Sensitive credentials *-credentials.yaml diff --git a/packages/create-app/src/createApp.ts b/packages/create-app/src/createApp.ts index 83584445b0..824d1d967e 100644 --- a/packages/create-app/src/createApp.ts +++ b/packages/create-app/src/createApp.ts @@ -50,15 +50,7 @@ export default async (opts: OptionValues): Promise => { return true; }, }, - { - type: 'list', - name: 'dbType', - message: chalk.blue('Select database for the backend [required]'), - choices: ['SQLite', 'PostgreSQL'], - }, ]); - answers.dbTypePG = answers.dbType === 'PostgreSQL'; - answers.dbTypeSqlite = answers.dbType === 'SQLite'; const templateDir = paths.resolveOwn('templates/default-app'); const tempDir = resolvePath(os.tmpdir(), answers.name); diff --git a/packages/create-app/templates/default-app/app-config.local.yaml b/packages/create-app/templates/default-app/app-config.local.yaml new file mode 100644 index 0000000000..976293b546 --- /dev/null +++ b/packages/create-app/templates/default-app/app-config.local.yaml @@ -0,0 +1 @@ +# Backstage override configuration for your local development environment diff --git a/packages/create-app/templates/default-app/app-config.production.yaml b/packages/create-app/templates/default-app/app-config.production.yaml index c180bbfdec..c6ae2c036b 100644 --- a/packages/create-app/templates/default-app/app-config.production.yaml +++ b/packages/create-app/templates/default-app/app-config.production.yaml @@ -17,3 +17,18 @@ backend: # backend will bind on the interface that corresponds to the backend.baseUrl # hostname. host: 0.0.0.0 + +# config options: https://node-postgres.com/api/client +database: + client: pg + connection: + host: ${POSTGRES_HOST} + port: ${POSTGRES_PORT} + user: ${POSTGRES_USER} + password: ${POSTGRES_PASSWORD} + # https://node-postgres.com/features/ssl + # you can set the sslmode configuration option via the `PGSSLMODE` environment variable + # see https://www.postgresql.org/docs/current/libpq-ssl.html Table 33.1. SSL Mode Descriptions (e.g. require) + # ssl: + # ca: # if you have a CA file and want to verify it you can uncomment this section + # $file: /ca/server.crt diff --git a/packages/create-app/templates/default-app/app-config.yaml.hbs b/packages/create-app/templates/default-app/app-config.yaml.hbs index e274ffeaf2..2bf366d756 100644 --- a/packages/create-app/templates/default-app/app-config.yaml.hbs +++ b/packages/create-app/templates/default-app/app-config.yaml.hbs @@ -26,27 +26,11 @@ backend: origin: http://localhost:3000 methods: [GET, POST, PUT, DELETE] credentials: true - {{#if dbTypeSqlite}} + # This is for local developement only, it is not recommended to use this in production + # The production database configuration is stored in app-config.production.yaml database: client: better-sqlite3 connection: ':memory:' - {{/if}} - {{#if dbTypePG}} - # config options: https://node-postgres.com/api/client - database: - client: pg - connection: - host: ${POSTGRES_HOST} - port: ${POSTGRES_PORT} - user: ${POSTGRES_USER} - password: ${POSTGRES_PASSWORD} - # https://node-postgres.com/features/ssl - # you can set the sslmode configuration option via the `PGSSLMODE` environment variable - # see https://www.postgresql.org/docs/current/libpq-ssl.html Table 33.1. SSL Mode Descriptions (e.g. require) - # ssl: - # ca: # if you have a CA file and want to verify it you can uncomment this section - # $file: /ca/server.crt - {{/if}} cache: store: memory # workingDirectory: /tmp # Use this to configure a working directory for the scaffolder, defaults to the OS temp-dir diff --git a/packages/create-app/templates/default-app/packages/backend/package.json.hbs b/packages/create-app/templates/default-app/packages/backend/package.json.hbs index 939b13bf86..acfd465b67 100644 --- a/packages/create-app/templates/default-app/packages/backend/package.json.hbs +++ b/packages/create-app/templates/default-app/packages/backend/package.json.hbs @@ -30,24 +30,18 @@ "@backstage/plugin-proxy-backend": "^{{version '@backstage/plugin-proxy-backend'}}", "@backstage/plugin-scaffolder-backend": "^{{version '@backstage/plugin-scaffolder-backend'}}", "@backstage/plugin-search-backend": "^{{version '@backstage/plugin-search-backend'}}", - {{#if dbTypePG}} "@backstage/plugin-search-backend-module-pg": "^{{version '@backstage/plugin-search-backend-module-pg'}}", - {{/if}} "@backstage/plugin-search-backend-node": "^{{version '@backstage/plugin-search-backend-node'}}", "@backstage/plugin-techdocs-backend": "^{{version '@backstage/plugin-techdocs-backend'}}", "dockerode": "^3.3.1", "express": "^4.17.1", "express-promise-router": "^4.1.0", "luxon": "^2.0.2", - {{#if dbTypePG}} "pg": "^8.3.0", - {{/if}} - {{#if dbTypeSqlite}} - "better-sqlite3": "^7.5.0", - {{/if}} "winston": "^3.2.1" }, "devDependencies": { + "better-sqlite3": "^7.5.0", "@backstage/cli": "^{{version '@backstage/cli'}}", "@types/dockerode": "^3.3.0", "@types/express": "^4.17.6", From 578148ba0e7b3256de7d7f115db663a28f668a35 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Fri, 22 Apr 2022 15:56:31 +0200 Subject: [PATCH 02/10] Remove postgres question from e2e tests Signed-off-by: Johan Haals --- .../default-app/packages/backend/package.json.hbs | 6 +++--- packages/e2e-test/src/commands/run.ts | 11 +---------- 2 files changed, 4 insertions(+), 13 deletions(-) diff --git a/packages/create-app/templates/default-app/packages/backend/package.json.hbs b/packages/create-app/templates/default-app/packages/backend/package.json.hbs index acfd465b67..c162dd880e 100644 --- a/packages/create-app/templates/default-app/packages/backend/package.json.hbs +++ b/packages/create-app/templates/default-app/packages/backend/package.json.hbs @@ -41,12 +41,12 @@ "winston": "^3.2.1" }, "devDependencies": { - "better-sqlite3": "^7.5.0", "@backstage/cli": "^{{version '@backstage/cli'}}", "@types/dockerode": "^3.3.0", - "@types/express": "^4.17.6", "@types/express-serve-static-core": "^4.17.5", - "@types/luxon": "^2.0.4" + "@types/express": "^4.17.6", + "@types/luxon": "^2.0.4", + "better-sqlite3": "^7.5.0" }, "files": [ "dist" diff --git a/packages/e2e-test/src/commands/run.ts b/packages/e2e-test/src/commands/run.ts index dc257c6f29..b6df9f61dd 100644 --- a/packages/e2e-test/src/commands/run.ts +++ b/packages/e2e-test/src/commands/run.ts @@ -51,7 +51,7 @@ export async function run() { const isPostgres = Boolean(process.env.POSTGRES_USER); print('Creating a Backstage App'); - const appDir = await createApp('test-app', isPostgres, workspaceDir, rootDir); + const appDir = await createApp('test-app', workspaceDir, rootDir); print('Creating a Backstage Plugin'); const pluginName = await createPlugin('test-plugin', appDir); @@ -191,7 +191,6 @@ async function pinYarnVersion(dir: string) { */ async function createApp( appName: string, - isPostgres: boolean, workspaceDir: string, rootDir: string, ) { @@ -215,14 +214,6 @@ async function createApp( await waitFor(() => stdout.includes('Enter a name for the app')); child.stdin?.write(`${appName}\n`); - await waitFor(() => stdout.includes('Select database for the backend')); - - if (isPostgres) { - // Simulate down arrow press - child.stdin?.write(`\u001B\u005B\u0042`); - } - child.stdin?.write(`\n`); - print('Waiting for app create script to be done'); await waitForExit(child); From ab17c3bbcdaf89185186da9a41efcecb515b75d4 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Tue, 26 Apr 2022 11:51:26 +0200 Subject: [PATCH 03/10] default search to Lunr Signed-off-by: Johan Haals --- .../packages/backend/src/plugins/search.ts.hbs | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/packages/create-app/templates/default-app/packages/backend/src/plugins/search.ts.hbs b/packages/create-app/templates/default-app/packages/backend/src/plugins/search.ts.hbs index 62bc3056cc..27c42fca31 100644 --- a/packages/create-app/templates/default-app/packages/backend/src/plugins/search.ts.hbs +++ b/packages/create-app/templates/default-app/packages/backend/src/plugins/search.ts.hbs @@ -4,9 +4,6 @@ import { IndexBuilder, LunrSearchEngine, } from '@backstage/plugin-search-backend-node'; -{{#if dbTypePG}} -import { PgSearchEngine } from '@backstage/plugin-search-backend-module-pg'; -{{/if}} import { PluginEnvironment } from '../types'; import { DefaultCatalogCollatorFactory } from '@backstage/plugin-catalog-backend'; import { DefaultTechDocsCollatorFactory } from '@backstage/plugin-techdocs-backend'; @@ -17,16 +14,9 @@ export default async function createPlugin( env: PluginEnvironment, ): Promise { // Initialize a connection to a search engine. - {{#if dbTypeSqlite}} const searchEngine = new LunrSearchEngine({ logger: env.logger, }); - {{/if}} - {{#if dbTypePG}} - const searchEngine = (await PgSearchEngine.supported(env.database)) - ? await PgSearchEngine.from({ database: env.database }) - : new LunrSearchEngine({ logger: env.logger }); - {{/if}} const indexBuilder = new IndexBuilder({ logger: env.logger, searchEngine, From 98ef64b45d63ab0e7293458cdb437894e6c80862 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Thu, 28 Apr 2022 14:21:37 +0200 Subject: [PATCH 04/10] test with postgres and sqlite Signed-off-by: Johan Haals --- packages/e2e-test/src/commands/run.ts | 60 +++++++++++++++++---------- 1 file changed, 37 insertions(+), 23 deletions(-) diff --git a/packages/e2e-test/src/commands/run.ts b/packages/e2e-test/src/commands/run.ts index b6df9f61dd..3e5b546225 100644 --- a/packages/e2e-test/src/commands/run.ts +++ b/packages/e2e-test/src/commands/run.ts @@ -21,6 +21,7 @@ import handlebars from 'handlebars'; import killTree from 'tree-kill'; import { resolve as resolvePath, join as joinPath } from 'path'; import puppeteer from 'puppeteer'; +import path from 'path'; import { spawnPiped, @@ -49,7 +50,6 @@ export async function run() { print('Building dist workspace'); const workspaceDir = await buildDistWorkspace('workspace', rootDir); - const isPostgres = Boolean(process.env.POSTGRES_USER); print('Creating a Backstage App'); const appDir = await createApp('test-app', workspaceDir, rootDir); @@ -62,8 +62,21 @@ export async function run() { print('Starting the app'); await testAppServe(pluginName, appDir); - print('Testing the backend startup'); - await testBackendStart(appDir, isPostgres); + const appConfig = path.resolve(appDir, 'app-config.yaml'); + if (Boolean(process.env.POSTGRES_USER)) { + print('Testing the PostgreSQL backend startup'); + await preCleanPostgres(); + const productionConfig = path.resolve(appDir, 'app-config.production.yaml'); + await testBackendStart( + appDir, + '--config', + appConfig, + '--config', + productionConfig, + ); + } + print('Testing the SQLite backend startup'); + await testBackendStart(appDir, '--config', appConfig); if (process.env.CI) { // Cleanup actually takes significant time, so skip it in CI since the @@ -100,8 +113,8 @@ async function buildDistWorkspace(workspaceName: string, rootDir: string) { } for (const pkgJsonPath of templatePackagePaths) { - const path = paths.resolveOwnRoot(pkgJsonPath); - const pkgTemplate = await fs.readFile(path, 'utf8'); + const jsonPath = paths.resolveOwnRoot(pkgJsonPath); + const pkgTemplate = await fs.readFile(jsonPath, 'utf8'); const pkg = JSON.parse( handlebars.compile(pkgTemplate)( { @@ -401,27 +414,28 @@ async function dropDB(database: string) { } } +/** Clean remnants from prior e2e runs */ +async function preCleanPostgres() { + print('Dropping old DBs'); + await Promise.all( + [ + 'catalog', + 'scaffolder', + 'auth', + 'identity', + 'proxy', + 'techdocs', + 'search', + ].map(name => dropDB(`backstage_plugin_${name}`)), + ); + print('Created DBs'); +} + /** * Start serving the newly created backend and make sure that all db migrations works correctly */ -async function testBackendStart(appDir: string, isPostgres: boolean) { - if (isPostgres) { - print('Dropping old DBs'); - await Promise.all( - [ - 'catalog', - 'scaffolder', - 'auth', - 'identity', - 'proxy', - 'techdocs', - 'search', - ].map(name => dropDB(`backstage_plugin_${name}`)), - ); - print('Created DBs'); - } - - const child = spawnPiped(['yarn', 'workspace', 'backend', 'start'], { +async function testBackendStart(appDir: string, ...args: string[]) { + const child = spawnPiped(['yarn', 'workspace', 'backend', 'start', ...args], { cwd: appDir, env: { ...process.env, From 882232ed03633474dafe6cee1abf39cb318b0e45 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Thu, 28 Apr 2022 14:49:42 +0200 Subject: [PATCH 05/10] Set postgres environment variables Signed-off-by: Johan Haals --- .github/workflows/verify_e2e-linux.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/verify_e2e-linux.yml b/.github/workflows/verify_e2e-linux.yml index e333aaba0c..daf041cd50 100644 --- a/.github/workflows/verify_e2e-linux.yml +++ b/.github/workflows/verify_e2e-linux.yml @@ -78,3 +78,5 @@ jobs: yarn e2e-test run env: BACKSTAGE_TEST_DISABLE_DOCKER: 1 + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres From a542839cfca273acee8e0958d070cd22334382f4 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Thu, 28 Apr 2022 15:07:09 +0200 Subject: [PATCH 06/10] e2e: Set postgres host address Signed-off-by: Johan Haals --- .github/workflows/verify_e2e-linux.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/verify_e2e-linux.yml b/.github/workflows/verify_e2e-linux.yml index daf041cd50..59f07e0442 100644 --- a/.github/workflows/verify_e2e-linux.yml +++ b/.github/workflows/verify_e2e-linux.yml @@ -80,3 +80,4 @@ jobs: BACKSTAGE_TEST_DISABLE_DOCKER: 1 POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres + POSTGRES_HOST: postgres From 3a31b00d7ba3c5c7aad33ddec239e111604b7321 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Fri, 29 Apr 2022 11:17:42 +0200 Subject: [PATCH 07/10] action: Specify postgres port Signed-off-by: Johan Haals --- .github/workflows/verify_e2e-linux.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/verify_e2e-linux.yml b/.github/workflows/verify_e2e-linux.yml index 59f07e0442..212149e415 100644 --- a/.github/workflows/verify_e2e-linux.yml +++ b/.github/workflows/verify_e2e-linux.yml @@ -81,3 +81,4 @@ jobs: POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres POSTGRES_HOST: postgres + POSTGRES_PORT: 5432 From 5eaa4197d0e680e6f800146b3ce0f6fe12ddedc9 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Fri, 29 Apr 2022 13:39:19 +0200 Subject: [PATCH 08/10] fix incorrect database indentation Signed-off-by: Johan Haals --- .../default-app/app-config.production.yaml | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/packages/create-app/templates/default-app/app-config.production.yaml b/packages/create-app/templates/default-app/app-config.production.yaml index c6ae2c036b..b9dd00bb01 100644 --- a/packages/create-app/templates/default-app/app-config.production.yaml +++ b/packages/create-app/templates/default-app/app-config.production.yaml @@ -18,17 +18,17 @@ backend: # hostname. host: 0.0.0.0 -# config options: https://node-postgres.com/api/client -database: - client: pg - connection: - host: ${POSTGRES_HOST} - port: ${POSTGRES_PORT} - user: ${POSTGRES_USER} - password: ${POSTGRES_PASSWORD} - # https://node-postgres.com/features/ssl - # you can set the sslmode configuration option via the `PGSSLMODE` environment variable - # see https://www.postgresql.org/docs/current/libpq-ssl.html Table 33.1. SSL Mode Descriptions (e.g. require) - # ssl: - # ca: # if you have a CA file and want to verify it you can uncomment this section - # $file: /ca/server.crt + # config options: https://node-postgres.com/api/client + database: + client: pg + connection: + host: ${POSTGRES_HOST} + port: ${POSTGRES_PORT} + user: ${POSTGRES_USER} + password: ${POSTGRES_PASSWORD} + # https://node-postgres.com/features/ssl + # you can set the sslmode configuration option via the `PGSSLMODE` environment variable + # see https://www.postgresql.org/docs/current/libpq-ssl.html Table 33.1. SSL Mode Descriptions (e.g. require) + # ssl: + # ca: # if you have a CA file and want to verify it you can uncomment this section + # $file: /ca/server.crt From 2bad9afe42038e524d22cbb214ae091d37e0f411 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Fri, 29 Apr 2022 13:56:16 +0200 Subject: [PATCH 09/10] e2e test: Fix e2e invalid configuration Signed-off-by: Johan Haals --- .changeset/smart-ghosts-search.md | 1 + .github/workflows/verify_e2e-linux.yml | 4 ++-- packages/e2e-test/src/commands/run.ts | 6 +++--- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.changeset/smart-ghosts-search.md b/.changeset/smart-ghosts-search.md index 5d8b3676cb..d1d32e71af 100644 --- a/.changeset/smart-ghosts-search.md +++ b/.changeset/smart-ghosts-search.md @@ -7,3 +7,4 @@ Removed the database choice from the `create-app` command. This reduces the step from development to production by always installing the dependencies and templating the production configuration in `app-config.production.yaml`. Added `app-config.local.yaml` to allow for local configuration overrides. +To replicate this behavior in an existing installation simply `touch app-config.local.yaml` in the project root and apply your local configuration. diff --git a/.github/workflows/verify_e2e-linux.yml b/.github/workflows/verify_e2e-linux.yml index 212149e415..f3e38dc68b 100644 --- a/.github/workflows/verify_e2e-linux.yml +++ b/.github/workflows/verify_e2e-linux.yml @@ -80,5 +80,5 @@ jobs: BACKSTAGE_TEST_DISABLE_DOCKER: 1 POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres - POSTGRES_HOST: postgres - POSTGRES_PORT: 5432 + POSTGRES_HOST: localhost + POSTGRES_PORT: ${{ job.services.postgres.ports[5432] }} diff --git a/packages/e2e-test/src/commands/run.ts b/packages/e2e-test/src/commands/run.ts index 3e5b546225..e1477d98d7 100644 --- a/packages/e2e-test/src/commands/run.ts +++ b/packages/e2e-test/src/commands/run.ts @@ -62,10 +62,10 @@ export async function run() { print('Starting the app'); await testAppServe(pluginName, appDir); - const appConfig = path.resolve(appDir, 'app-config.yaml'); if (Boolean(process.env.POSTGRES_USER)) { print('Testing the PostgreSQL backend startup'); await preCleanPostgres(); + const appConfig = path.resolve(appDir, 'app-config.yaml'); const productionConfig = path.resolve(appDir, 'app-config.production.yaml'); await testBackendStart( appDir, @@ -76,7 +76,7 @@ export async function run() { ); } print('Testing the SQLite backend startup'); - await testBackendStart(appDir, '--config', appConfig); + await testBackendStart(appDir); if (process.env.CI) { // Cleanup actually takes significant time, so skip it in CI since the @@ -408,7 +408,7 @@ async function dropDB(database: string) { }; try { - await pgtools.dropdb({ config }, database); + await pgtools.dropdb(config, database); } catch (_) { /* do nothing*/ } From 44f2c43a426d40db83f19f088daeb10fa01746bb Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Mon, 2 May 2022 16:59:53 +0200 Subject: [PATCH 10/10] mention sqlite in changeset Signed-off-by: Johan Haals --- .changeset/smart-ghosts-search.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.changeset/smart-ghosts-search.md b/.changeset/smart-ghosts-search.md index d1d32e71af..3fb3c9c716 100644 --- a/.changeset/smart-ghosts-search.md +++ b/.changeset/smart-ghosts-search.md @@ -8,3 +8,21 @@ This reduces the step from development to production by always installing the de Added `app-config.local.yaml` to allow for local configuration overrides. To replicate this behavior in an existing installation simply `touch app-config.local.yaml` in the project root and apply your local configuration. + +`better-sqlite3` has been moved to devDependencies, for existing installations using postgres in production and SQLite in development it's recommended to move SQLite into the devDependencies section to avoid unnecessary dependencies during builds. + +in `packages/backend/package.json` + +```diff + "dependencies": { + ... + "pg": "^8.3.0", +- "better-sqlite3": "^7.5.0", + "winston": "^3.2.1" + }, + "devDependencies": { + ... + "@types/luxon": "^2.0.4", ++ "better-sqlite3": "^7.5.0" + } +```