From ae903f8e718d89cdff5538bf2d5c93ec7c59c360 Mon Sep 17 00:00:00 2001 From: Tavis Aitken Date: Tue, 27 Apr 2021 11:01:56 -0600 Subject: [PATCH 01/93] Added config schema to the Splunk On Call plugin. Signed-off-by: Tavis Aitken --- .changeset/fresh-vans-nail.md | 5 +++++ plugins/splunk-on-call/package.json | 6 ++++-- plugins/splunk-on-call/schema.d.ts | 26 ++++++++++++++++++++++++++ 3 files changed, 35 insertions(+), 2 deletions(-) create mode 100644 .changeset/fresh-vans-nail.md create mode 100644 plugins/splunk-on-call/schema.d.ts diff --git a/.changeset/fresh-vans-nail.md b/.changeset/fresh-vans-nail.md new file mode 100644 index 0000000000..64af4ee60b --- /dev/null +++ b/.changeset/fresh-vans-nail.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-splunk-on-call': patch +--- + +Added config schema to expose `splunkOnCall.eventsRestEndpoint` config option to the frontend diff --git a/plugins/splunk-on-call/package.json b/plugins/splunk-on-call/package.json index 9e93004bf5..b033b41e75 100644 --- a/plugins/splunk-on-call/package.json +++ b/plugins/splunk-on-call/package.json @@ -59,6 +59,8 @@ "node-fetch": "^2.6.1" }, "files": [ - "dist" - ] + "dist", + "schema.d.ts" + ], + "configSchema": "schema.d.ts" } diff --git a/plugins/splunk-on-call/schema.d.ts b/plugins/splunk-on-call/schema.d.ts new file mode 100644 index 0000000000..b14f2ef110 --- /dev/null +++ b/plugins/splunk-on-call/schema.d.ts @@ -0,0 +1,26 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +export interface Config { + /** + * Splunk On Call Plugin specific configs + */ + splunkOnCall: { + /** + * @visibility frontend + */ + eventsRestEndpoint: string; + }; +} From d2329e7486ee0507921290289399a979cce03d05 Mon Sep 17 00:00:00 2001 From: Tejas Kumar Date: Thu, 20 May 2021 16:23:13 +0200 Subject: [PATCH 02/93] Use stringifyEntityRef in place of deprecated Signed-off-by: Tejas Kumar --- plugins/techdocs-backend/src/DocsBuilder/builder.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/plugins/techdocs-backend/src/DocsBuilder/builder.ts b/plugins/techdocs-backend/src/DocsBuilder/builder.ts index d1aa1e08be..4c4744879c 100644 --- a/plugins/techdocs-backend/src/DocsBuilder/builder.ts +++ b/plugins/techdocs-backend/src/DocsBuilder/builder.ts @@ -16,7 +16,7 @@ import { Entity, ENTITY_DEFAULT_NAMESPACE, - serializeEntityRef, + stringifyEntityRef, } from '@backstage/catalog-model'; import { NotModifiedError } from '@backstage/errors'; import { @@ -75,7 +75,7 @@ export class DocsBuilder { */ this.logger.info( - `Step 1 of 3: Preparing docs for entity ${serializeEntityRef( + `Step 1 of 3: Preparing docs for entity ${stringifyEntityRef( this.entity, )}`, ); @@ -116,7 +116,7 @@ export class DocsBuilder { // Set last check happened to now new BuildMetadataStorage(this.entity.metadata.uid).setLastUpdated(); this.logger.debug( - `Docs for ${serializeEntityRef( + `Docs for ${stringifyEntityRef( this.entity, )} are unmodified. Using cache, skipping generate and prepare`, ); @@ -126,7 +126,7 @@ export class DocsBuilder { } this.logger.info( - `Prepare step completed for entity ${serializeEntityRef( + `Prepare step completed for entity ${stringifyEntityRef( this.entity, )}, stored at ${preparedDir}`, ); @@ -136,7 +136,7 @@ export class DocsBuilder { */ this.logger.info( - `Step 2 of 3: Generating docs for entity ${serializeEntityRef( + `Step 2 of 3: Generating docs for entity ${stringifyEntityRef( this.entity, )}`, ); @@ -176,7 +176,7 @@ export class DocsBuilder { */ this.logger.info( - `Step 3 of 3: Publishing docs for entity ${serializeEntityRef( + `Step 3 of 3: Publishing docs for entity ${stringifyEntityRef( this.entity, )}`, ); From a345a2e019e0f33210cd72772c4990624644e093 Mon Sep 17 00:00:00 2001 From: Tejas Kumar Date: Thu, 20 May 2021 16:32:31 +0200 Subject: [PATCH 03/93] Use backend.workingDir for techdocs Signed-off-by: Tejas Kumar --- plugins/techdocs-backend/src/DocsBuilder/builder.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/plugins/techdocs-backend/src/DocsBuilder/builder.ts b/plugins/techdocs-backend/src/DocsBuilder/builder.ts index 4c4744879c..8495e88d37 100644 --- a/plugins/techdocs-backend/src/DocsBuilder/builder.ts +++ b/plugins/techdocs-backend/src/DocsBuilder/builder.ts @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +import { getRootLogger, loadBackendConfig } from '@backstage/backend-common'; import { Entity, ENTITY_DEFAULT_NAMESPACE, @@ -142,7 +143,12 @@ export class DocsBuilder { ); // Create a temporary directory to store the generated files in. - const tmpdirPath = os.tmpdir(); + const config = await loadBackendConfig({ + argv: process.argv, + logger: getRootLogger(), + }); + const workingDir = config.get('backend.workingDirectory'); + const tmpdirPath = workingDir ? String(workingDir) : os.tmpdir(); // Fixes a problem with macOS returning a path that is a symlink const tmpdirResolvedPath = fs.realpathSync(tmpdirPath); const outputDir = await fs.mkdtemp( From d93a3f98533e496fc5a1f045f4deb49b5e8e7c53 Mon Sep 17 00:00:00 2001 From: Tejas Kumar Date: Thu, 20 May 2021 16:34:04 +0200 Subject: [PATCH 04/93] Refactor to use optional string Signed-off-by: Tejas Kumar --- plugins/techdocs-backend/src/DocsBuilder/builder.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/techdocs-backend/src/DocsBuilder/builder.ts b/plugins/techdocs-backend/src/DocsBuilder/builder.ts index 8495e88d37..e43ed72c31 100644 --- a/plugins/techdocs-backend/src/DocsBuilder/builder.ts +++ b/plugins/techdocs-backend/src/DocsBuilder/builder.ts @@ -147,8 +147,8 @@ export class DocsBuilder { argv: process.argv, logger: getRootLogger(), }); - const workingDir = config.get('backend.workingDirectory'); - const tmpdirPath = workingDir ? String(workingDir) : os.tmpdir(); + const workingDir = config.getOptionalString('backend.workingDirectory'); + const tmpdirPath = workingDir || os.tmpdir(); // Fixes a problem with macOS returning a path that is a symlink const tmpdirResolvedPath = fs.realpathSync(tmpdirPath); const outputDir = await fs.mkdtemp( From 6013a16dc37ac5a0d2b61b650e05f993dca0cc31 Mon Sep 17 00:00:00 2001 From: Tejas Kumar Date: Thu, 20 May 2021 16:35:11 +0200 Subject: [PATCH 05/93] Add Changeset Signed-off-by: Tejas Kumar --- .changeset/nasty-wasps-look.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/nasty-wasps-look.md diff --git a/.changeset/nasty-wasps-look.md b/.changeset/nasty-wasps-look.md new file mode 100644 index 0000000000..6de02cbb04 --- /dev/null +++ b/.changeset/nasty-wasps-look.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-techdocs-backend': patch +--- + +TechDocs: Support configurable working directory as temp dir From 35cb3c21cf789e77bdbf20532ee83c1a5a3b9ab4 Mon Sep 17 00:00:00 2001 From: RISHABH BUDHIRAJA Date: Fri, 21 May 2021 01:36:28 +0530 Subject: [PATCH 06/93] FIx: Diagram component using hardcoded namespace Signed-off-by: RISHABH BUDHIRAJA --- .../src/components/SystemDiagramCard/SystemDiagramCard.tsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/catalog/src/components/SystemDiagramCard/SystemDiagramCard.tsx b/plugins/catalog/src/components/SystemDiagramCard/SystemDiagramCard.tsx index 5fd32a6298..42d493912c 100644 --- a/plugins/catalog/src/components/SystemDiagramCard/SystemDiagramCard.tsx +++ b/plugins/catalog/src/components/SystemDiagramCard/SystemDiagramCard.tsx @@ -149,6 +149,7 @@ export function SystemDiagramCard() { const currentSystemNode = stringifyEntityRef(entity); const systemNodes = new Array<{ id: string; kind: string; name: string }>(); const systemEdges = new Array<{ from: string; to: string; label: string }>(); + const ref = parseEntityRef(currentSystemNode); const catalogApi = useApi(catalogApiRef); const { loading, error, value: catalogResponse } = useAsync(() => { @@ -157,7 +158,7 @@ export function SystemDiagramCard() { kind: ['Component', 'API', 'Resource', 'System', 'Domain'], 'spec.system': [ currentSystemName, - `${ENTITY_DEFAULT_NAMESPACE}/${currentSystemName}`, + `${ref.namespace || 'Current Namespace'}/${currentSystemName}`, ], }, }); From 804a605ad84593411c69d9729c301a867987e0d0 Mon Sep 17 00:00:00 2001 From: RISHABH BUDHIRAJA Date: Wed, 26 May 2021 01:20:05 +0530 Subject: [PATCH 07/93] Revert "FIx: Diagram component using hardcoded namespace" This reverts commit dd3fc3e6616ab801e7fc1c9deeb7270f07cb3892. Signed-off-by: RISHABH BUDHIRAJA --- .../src/components/SystemDiagramCard/SystemDiagramCard.tsx | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/plugins/catalog/src/components/SystemDiagramCard/SystemDiagramCard.tsx b/plugins/catalog/src/components/SystemDiagramCard/SystemDiagramCard.tsx index 42d493912c..5fd32a6298 100644 --- a/plugins/catalog/src/components/SystemDiagramCard/SystemDiagramCard.tsx +++ b/plugins/catalog/src/components/SystemDiagramCard/SystemDiagramCard.tsx @@ -149,7 +149,6 @@ export function SystemDiagramCard() { const currentSystemNode = stringifyEntityRef(entity); const systemNodes = new Array<{ id: string; kind: string; name: string }>(); const systemEdges = new Array<{ from: string; to: string; label: string }>(); - const ref = parseEntityRef(currentSystemNode); const catalogApi = useApi(catalogApiRef); const { loading, error, value: catalogResponse } = useAsync(() => { @@ -158,7 +157,7 @@ export function SystemDiagramCard() { kind: ['Component', 'API', 'Resource', 'System', 'Domain'], 'spec.system': [ currentSystemName, - `${ref.namespace || 'Current Namespace'}/${currentSystemName}`, + `${ENTITY_DEFAULT_NAMESPACE}/${currentSystemName}`, ], }, }); From 6a584214360aaff67670bde6ccfb7d9738dd3792 Mon Sep 17 00:00:00 2001 From: RISHABH BUDHIRAJA Date: Wed, 26 May 2021 01:20:59 +0530 Subject: [PATCH 08/93] Fix: Diagram component using hardcoded namespace Signed-off-by: RISHABH BUDHIRAJA --- .../src/components/SystemDiagramCard/SystemDiagramCard.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/catalog/src/components/SystemDiagramCard/SystemDiagramCard.tsx b/plugins/catalog/src/components/SystemDiagramCard/SystemDiagramCard.tsx index 5fd32a6298..4e34fb1910 100644 --- a/plugins/catalog/src/components/SystemDiagramCard/SystemDiagramCard.tsx +++ b/plugins/catalog/src/components/SystemDiagramCard/SystemDiagramCard.tsx @@ -157,7 +157,7 @@ export function SystemDiagramCard() { kind: ['Component', 'API', 'Resource', 'System', 'Domain'], 'spec.system': [ currentSystemName, - `${ENTITY_DEFAULT_NAMESPACE}/${currentSystemName}`, + `${entity.metadata.namespace || ENTITY_DEFAULT_NAMESPACE}/${currentSystemName}`, ], }, }); From d2d42a7fa29ed9d83b750e5177b5606d725a943b Mon Sep 17 00:00:00 2001 From: RISHABH BUDHIRAJA Date: Wed, 26 May 2021 18:44:15 +0530 Subject: [PATCH 09/93] adds changeset for patch Signed-off-by: RISHABH BUDHIRAJA --- .changeset/thick-donkeys-fold.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/thick-donkeys-fold.md diff --git a/.changeset/thick-donkeys-fold.md b/.changeset/thick-donkeys-fold.md new file mode 100644 index 0000000000..4720439479 --- /dev/null +++ b/.changeset/thick-donkeys-fold.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog': patch +--- + +Fix for Diagram component using hardcoded namespace From 2e1fbe203be32e9fcf717ad483e8c7200c4d3df0 Mon Sep 17 00:00:00 2001 From: Anastasia Rodionova Date: Tue, 8 Jun 2021 19:45:13 +0200 Subject: [PATCH 10/93] Do not add / for html pages in rewriteDocLinks Signed-off-by: Anastasia Rodionova --- .changeset/seven-wolves-clean.md | 5 +++++ .../techdocs/src/reader/transformers/rewriteDocLinks.test.ts | 1 + plugins/techdocs/src/reader/transformers/rewriteDocLinks.ts | 2 +- 3 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 .changeset/seven-wolves-clean.md diff --git a/.changeset/seven-wolves-clean.md b/.changeset/seven-wolves-clean.md new file mode 100644 index 0000000000..3b807b1af6 --- /dev/null +++ b/.changeset/seven-wolves-clean.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-techdocs': patch +--- + +Do not add trailing slash for .html pages during doc links rewriting diff --git a/plugins/techdocs/src/reader/transformers/rewriteDocLinks.test.ts b/plugins/techdocs/src/reader/transformers/rewriteDocLinks.test.ts index 01467f665d..4ebd7411ac 100644 --- a/plugins/techdocs/src/reader/transformers/rewriteDocLinks.test.ts +++ b/plugins/techdocs/src/reader/transformers/rewriteDocLinks.test.ts @@ -82,6 +82,7 @@ describe('normalizeUrl', () => { ['http://example.org/folder#intro', 'http://example.org/folder/#intro'], ['http://example.org/folder/#intro', 'http://example.org/folder/#intro'], ['http://example.org/folder#', 'http://example.org/folder/#'], + ['http://example.org/page.html', 'http://example.org/page.html'], ])('should handle %s', (url, expected) => { expect(normalizeUrl(url)).toEqual(expected); }); diff --git a/plugins/techdocs/src/reader/transformers/rewriteDocLinks.ts b/plugins/techdocs/src/reader/transformers/rewriteDocLinks.ts index 8c4bc29201..8d8fafa454 100644 --- a/plugins/techdocs/src/reader/transformers/rewriteDocLinks.ts +++ b/plugins/techdocs/src/reader/transformers/rewriteDocLinks.ts @@ -58,7 +58,7 @@ export const rewriteDocLinks = (): Transformer => { export function normalizeUrl(input: string): string { const url = new URL(input); - if (!url.pathname.endsWith('/')) { + if (!url.pathname.endsWith('/') && !url.pathname.endsWith('.html')) { url.pathname += '/'; } From 14ce64b4fb338298577ac2772317a762d4de0d5f Mon Sep 17 00:00:00 2001 From: Anastasia Rodionova Date: Wed, 9 Jun 2021 18:08:01 +0200 Subject: [PATCH 11/93] Add pagination to ApisExplorerTable Signed-off-by: Anastasia Rodionova --- .changeset/shaggy-vans-travel.md | 5 +++++ .../src/components/ApiExplorerTable/ApiExplorerTable.tsx | 4 +++- 2 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 .changeset/shaggy-vans-travel.md diff --git a/.changeset/shaggy-vans-travel.md b/.changeset/shaggy-vans-travel.md new file mode 100644 index 0000000000..898731f584 --- /dev/null +++ b/.changeset/shaggy-vans-travel.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-api-docs': patch +--- + +Add pagination to ApiExplorerTable diff --git a/plugins/api-docs/src/components/ApiExplorerTable/ApiExplorerTable.tsx b/plugins/api-docs/src/components/ApiExplorerTable/ApiExplorerTable.tsx index 5b9808e26d..78b5f1d240 100644 --- a/plugins/api-docs/src/components/ApiExplorerTable/ApiExplorerTable.tsx +++ b/plugins/api-docs/src/components/ApiExplorerTable/ApiExplorerTable.tsx @@ -199,7 +199,9 @@ export const ApiExplorerTable = ({ isLoading={loading} columns={columns} options={{ - paging: false, + paging: true, + pageSize: 20, + pageSizeOptions: [20, 50, 100], actionsColumnIndex: -1, loadingType: 'linear', padding: 'dense', From 772dbdb51145f891a21b35ee61b6a71a525c9d25 Mon Sep 17 00:00:00 2001 From: Minn Soe Date: Mon, 17 May 2021 18:16:06 +0100 Subject: [PATCH 12/93] feat: add database manager with per plugin config This commit introduces: - a new backwards compatible database manager which allows the end-user to set global and per plugin database configuration. - an early iteration of a database connector interface. - provides helper functions for each of the database connectors to meet the new interface. Signed-off-by: Minn Soe --- .changeset/five-donkeys-brake.md | 50 ++ .../tutorials/configuring-plugin-databases.md | 107 ++++ packages/backend-common/api-report.md | 459 ++++++++++-------- packages/backend-common/config.d.ts | 29 ++ .../src/database/PluginConnection.test.ts | 325 +++++++++++++ .../src/database/PluginConnection.ts | 160 ++++++ .../src/database/connection.test.ts | 51 +- .../backend-common/src/database/connection.ts | 101 +++- .../backend-common/src/database/connector.ts | 30 ++ packages/backend-common/src/database/index.ts | 1 + packages/backend-common/src/database/mysql.ts | 21 + .../backend-common/src/database/postgres.ts | 21 + .../backend-common/src/database/sqlite3.ts | 26 + packages/backend/src/index.ts | 4 +- 14 files changed, 1159 insertions(+), 226 deletions(-) create mode 100644 .changeset/five-donkeys-brake.md create mode 100644 docs/tutorials/configuring-plugin-databases.md create mode 100644 packages/backend-common/src/database/PluginConnection.test.ts create mode 100644 packages/backend-common/src/database/PluginConnection.ts create mode 100644 packages/backend-common/src/database/connector.ts diff --git a/.changeset/five-donkeys-brake.md b/.changeset/five-donkeys-brake.md new file mode 100644 index 0000000000..c9db3112a9 --- /dev/null +++ b/.changeset/five-donkeys-brake.md @@ -0,0 +1,50 @@ +--- +'example-backend': minor +'@backstage/backend-common': minor +--- + +Introduces `PluginConnectionDatabaseManager`, a backwards compatible database +connection manager which allows developers to configure database connections on +a per plugin basis. + +The `backend.database` config path allows you to set `prefix` to use an +alternate prefix for automatically generated database names, the default is +`backstage_plugin_`. Use `backend.database.plugin.` to set plugin +specific database connection configuration, e.g. + +```yaml +backend: + database: + client: 'pg', + prefix: 'custom_prefix_' + connection: + host: 'localhost' + user: 'foo' + password: 'bar' + plugin: + catalog: + connection: + database: 'database_name_overriden' + scaffolder: + client: 'sqlite3' + connection: ':inmemory' +``` + +Existing backstage installations can be migrated by swapping out the database +manager under `packages/backend/src/index.ts` as shown below: + +```diff +import { +- SingleConnectionDatabaseManager, ++ PluginConnectionDatabaseManager, +} from '@backstage/backend-common'; + +// ... + +function makeCreateEnv(config: Config) { + // ... +- const databaseManager = SingleConnectionDatabaseManager.fromConfig(config); ++ const databaseManager = PluginConnectionDatabaseManager.fromConfig(config); + // ... +} +``` diff --git a/docs/tutorials/configuring-plugin-databases.md b/docs/tutorials/configuring-plugin-databases.md new file mode 100644 index 0000000000..b6281cb0ae --- /dev/null +++ b/docs/tutorials/configuring-plugin-databases.md @@ -0,0 +1,107 @@ +--- +id: configuring-plugin-databases +title: Configuring Plugin Specific Databases +# prettier-ignore +description: Guide on how to use predefined databases for each plugin. +--- + +There are occasions where it may be difficult to deploy Backstage with +automatically created databases in production due to access control or other +restrictions. For example, your infrastructure might be defined as code using +tools such as Terraform or AWS CloudFormation where the name of each database is +defined, created and assigned explicitly. + +`@backstage/backend-common` provides an alternate database manager which allows +you to set the client and database connection on a per plugin basis. This means +that you can do selectively run certain plugins in memory with `sqlite3`, set +different connection config including the name of the database and more. + +There are two additional configuration options for this database manager: + +- **`backend.database.prefix`:** is used to override the default + `backstage_plugin_` prefix which is used to generate a database name when it + is not explicitly set for that plugin. +- **`backend.database.plugin.`:** is used to define a `client` and + `connection` block for the plugin matching the `pluginId`, e.g. `catalog` is + the `pluginId` for the catalog plugin and any configuration defined under that + block is specific to that plugin. + +## Install Database Drivers + +If you intend to use both `postgres` and `sqlite3`, you need to make sure the +appropriate database drivers are installed in your `backend` package. + +```shell +cd packages/backend + +# install pg if you need postgres +yarn add pg + +# install sqlite3 if you intend to set it as the client +yarn add sqlite3 +``` + +## Add Configuration + +To override the default prefix, `backstage_plugin_`, set +`backend.database.prefix` as shown below. This will use databases such as +`my_company_catalog` and `my_company_auth` instead of `backstage_plugin_catalog` +and `backstage_plugin_auth`. + +```yaml +backend: + database: + client: pg + prefix: my_company_ + connection: + host: localhost + user: postgres + password: password + plugin: + code-coverage: + connection: + database: pg_code_coverage_set_by_user +``` + +In the example above, the `code-coverage` plugin will use the same connection +configuration defined under `database.connection` and use +`pg_code_coverage_set_by_user` instead of `my_company_code-coverage` which would +be automatically generated if a plugin configuration wasn't explicitly set. + +## Integrate `PluginConnectionDatabaseManager` into `backend` + +The `SingleConnectionDatabaseManager` used by default should be replaced with +the `PluginConnectionDatabaseManager` in your `packages/backend/src/index.ts` +file. Import the manager and replace the `.fromConfig` call as shown below: + +```diff +import { +- SingleConnectionDatabaseManager, ++ PluginConnectionDatabaseManager, +} from '@backstage/backend-common'; + +// ... + +function makeCreateEnv(config: Config) { + // ... +- const databaseManager = SingleConnectionDatabaseManager.fromConfig(config); ++ const databaseManager = PluginConnectionDatabaseManager.fromConfig(config); + // ... +} +``` + +## Check Your Databases + +The `PluginConnectionDatabaseManager` preserves the behaviour of the +`SingleConnectionDatabaseManager`. If the database does not exist, it will +attempt to create it. You should ensure the databases that you configure exists +and that the connection details have the appropriate permissions to work with +each of the given databases if you are using this database manager to set the +database name upfront. If each database needs its own connection username, +password or host - you may set them under the plugin's `connection` block. + +`sqlite3` databases do not need to be created upfront as with the existing +database manager. + +Your Backstage App can now use different database clients and configuration per +plugin! diff --git a/packages/backend-common/api-report.md b/packages/backend-common/api-report.md index 068220cad3..2648e7af13 100644 --- a/packages/backend-common/api-report.md +++ b/packages/backend-common/api-report.md @@ -3,7 +3,6 @@ > Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). ```ts - import { AzureIntegration } from '@backstage/integration'; import { BitbucketIntegration } from '@backstage/integration'; import { Config } from '@backstage/config'; @@ -31,95 +30,120 @@ import { Writable } from 'stream'; // @public (undocumented) export class AzureUrlReader implements UrlReader { - constructor(integration: AzureIntegration, deps: { - treeResponseFactory: ReadTreeResponseFactory; - }); - // (undocumented) - static factory: ReaderFactory; - // (undocumented) - read(url: string): Promise; - // (undocumented) - readTree(url: string, options?: ReadTreeOptions): Promise; - // (undocumented) - search(url: string, options?: SearchOptions): Promise; - // (undocumented) - toString(): string; + constructor( + integration: AzureIntegration, + deps: { + treeResponseFactory: ReadTreeResponseFactory; + }, + ); + // (undocumented) + static factory: ReaderFactory; + // (undocumented) + read(url: string): Promise; + // (undocumented) + readTree(url: string, options?: ReadTreeOptions): Promise; + // (undocumented) + search(url: string, options?: SearchOptions): Promise; + // (undocumented) + toString(): string; } // @public export class BitbucketUrlReader implements UrlReader { - constructor(integration: BitbucketIntegration, deps: { - treeResponseFactory: ReadTreeResponseFactory; - }); - // (undocumented) - static factory: ReaderFactory; - // (undocumented) - read(url: string): Promise; - // (undocumented) - readTree(url: string, options?: ReadTreeOptions): Promise; - // (undocumented) - search(url: string, options?: SearchOptions): Promise; - // (undocumented) - toString(): string; + constructor( + integration: BitbucketIntegration, + deps: { + treeResponseFactory: ReadTreeResponseFactory; + }, + ); + // (undocumented) + static factory: ReaderFactory; + // (undocumented) + read(url: string): Promise; + // (undocumented) + readTree(url: string, options?: ReadTreeOptions): Promise; + // (undocumented) + search(url: string, options?: SearchOptions): Promise; + // (undocumented) + toString(): string; } // @public export interface CacheClient { - delete(key: string): Promise; - get(key: string): Promise; - set(key: string, value: JsonValue, options?: CacheSetOptions): Promise; + delete(key: string): Promise; + get(key: string): Promise; + set(key: string, value: JsonValue, options?: CacheSetOptions): Promise; } // @public export class CacheManager { - forPlugin(pluginId: string): PluginCacheManager; - static fromConfig(config: Config, options?: CacheManagerOptions): CacheManager; - } + forPlugin(pluginId: string): PluginCacheManager; + static fromConfig(config: Config, options?: CacheManagerOptions): CacheManager; +} // @public (undocumented) export const coloredFormat: winston.Logform.Format; // @public (undocumented) export interface ContainerRunner { - // (undocumented) - runContainer(opts: RunContainerOptions): Promise; + // (undocumented) + runContainer(opts: RunContainerOptions): Promise; } // @public @deprecated export const createDatabase: typeof createDatabaseClient; // @public -export function createDatabaseClient(dbConfig: Config, overrides?: Partial): Knex; +export function createDatabaseClient( + dbConfig: Config, + overrides?: Partial, +): Knex; // @public (undocumented) -export function createRootLogger(options?: winston.LoggerOptions, env?: NodeJS.ProcessEnv): winston.Logger; +export function createRootLogger( + options?: winston.LoggerOptions, + env?: NodeJS.ProcessEnv, +): winston.Logger; // @public export function createServiceBuilder(_module: NodeModule): ServiceBuilderImpl; // @public (undocumented) -export function createStatusCheckRouter(options: StatusCheckRouterOptions): Promise; +export function createStatusCheckRouter( + options: StatusCheckRouterOptions, +): Promise; // @public (undocumented) export class DockerContainerRunner implements ContainerRunner { - constructor({ dockerClient }: { - dockerClient: Docker; - }); - // (undocumented) - runContainer({ imageName, command, args, logStream, mountDirs, workingDir, envVars, }: RunContainerOptions): Promise; + constructor({ dockerClient }: { dockerClient: Docker }); + // (undocumented) + runContainer({ + imageName, + command, + args, + logStream, + mountDirs, + workingDir, + envVars, + }: RunContainerOptions): Promise; } // @public -export function ensureDatabaseExists(dbConfig: Config, ...databases: Array): Promise; +export function ensureDatabaseExists( + dbConfig: Config, + ...databases: Array +): Promise; // @public -export function errorHandler(options?: ErrorHandlerOptions): ErrorRequestHandler; +export function errorHandler( + options?: ErrorHandlerOptions, +): ErrorRequestHandler; // @public (undocumented) export type ErrorHandlerOptions = { - showStackTraces?: boolean; - logger?: Logger; - logClientErrors?: boolean; + showStackTraces?: boolean; + logger?: Logger; + logClientErrors?: boolean; }; // @public (undocumented) @@ -130,120 +154,141 @@ export function getVoidLogger(): winston.Logger; // @public (undocumented) export class Git { - // (undocumented) - add({ dir, filepath, }: { - dir: string; - filepath: string; - }): Promise; - // (undocumented) - addRemote({ dir, url, remote, }: { - dir: string; - remote: string; - url: string; - }): Promise; - // (undocumented) - clone({ url, dir, ref, }: { - url: string; - dir: string; - ref?: string; - }): Promise; - // (undocumented) - commit({ dir, message, author, committer, }: { - dir: string; - message: string; - author: { - name: string; - email: string; - }; - committer: { - name: string; - email: string; - }; - }): Promise; - // (undocumented) - currentBranch({ dir, fullName, }: { - dir: string; - fullName?: boolean; - }): Promise; - // (undocumented) - fetch({ dir, remote, }: { - dir: string; - remote?: string; - }): Promise; - // (undocumented) - static fromAuth: ({ username, password, logger, }: { - username?: string | undefined; - password?: string | undefined; - logger?: Logger | undefined; - }) => Git; - // (undocumented) - init({ dir }: { - dir: string; - }): Promise; - // (undocumented) - merge({ dir, theirs, ours, author, committer, }: { - dir: string; - theirs: string; - ours?: string; - author: { - name: string; - email: string; - }; - committer: { - name: string; - email: string; - }; - }): Promise; - // (undocumented) - push({ dir, remote }: { - dir: string; - remote: string; - }): Promise; - // (undocumented) - readCommit({ dir, sha, }: { - dir: string; - sha: string; - }): Promise; - // (undocumented) - resolveRef({ dir, ref, }: { - dir: string; - ref: string; - }): Promise; + // (undocumented) + add({ dir, filepath }: { dir: string; filepath: string }): Promise; + // (undocumented) + addRemote({ + dir, + url, + remote, + }: { + dir: string; + remote: string; + url: string; + }): Promise; + // (undocumented) + clone({ + url, + dir, + ref, + }: { + url: string; + dir: string; + ref?: string; + }): Promise; + // (undocumented) + commit({ + dir, + message, + author, + committer, + }: { + dir: string; + message: string; + author: { + name: string; + email: string; + }; + committer: { + name: string; + email: string; + }; + }): Promise; + // (undocumented) + currentBranch({ + dir, + fullName, + }: { + dir: string; + fullName?: boolean; + }): Promise; + // (undocumented) + fetch({ dir, remote }: { dir: string; remote?: string }): Promise; + // (undocumented) + static fromAuth: ({ + username, + password, + logger, + }: { + username?: string | undefined; + password?: string | undefined; + logger?: Logger | undefined; + }) => Git; + // (undocumented) + init({ dir }: { dir: string }): Promise; + // (undocumented) + merge({ + dir, + theirs, + ours, + author, + committer, + }: { + dir: string; + theirs: string; + ours?: string; + author: { + name: string; + email: string; + }; + committer: { + name: string; + email: string; + }; + }): Promise; + // (undocumented) + push({ dir, remote }: { dir: string; remote: string }): Promise; + // (undocumented) + readCommit({ + dir, + sha, + }: { + dir: string; + sha: string; + }): Promise; + // (undocumented) + resolveRef({ dir, ref }: { dir: string; ref: string }): Promise; } // @public export class GithubUrlReader implements UrlReader { - constructor(integration: GitHubIntegration, deps: { - treeResponseFactory: ReadTreeResponseFactory; - credentialsProvider: GithubCredentialsProvider; - }); - // (undocumented) - static factory: ReaderFactory; - // (undocumented) - read(url: string): Promise; - // (undocumented) - readTree(url: string, options?: ReadTreeOptions): Promise; - // (undocumented) - search(url: string, options?: SearchOptions): Promise; - // (undocumented) - toString(): string; + constructor( + integration: GitHubIntegration, + deps: { + treeResponseFactory: ReadTreeResponseFactory; + credentialsProvider: GithubCredentialsProvider; + }, + ); + // (undocumented) + static factory: ReaderFactory; + // (undocumented) + read(url: string): Promise; + // (undocumented) + readTree(url: string, options?: ReadTreeOptions): Promise; + // (undocumented) + search(url: string, options?: SearchOptions): Promise; + // (undocumented) + toString(): string; } // @public (undocumented) export class GitlabUrlReader implements UrlReader { - constructor(integration: GitLabIntegration, deps: { - treeResponseFactory: ReadTreeResponseFactory; - }); - // (undocumented) - static factory: ReaderFactory; - // (undocumented) - read(url: string): Promise; - // (undocumented) - readTree(url: string, options?: ReadTreeOptions): Promise; - // (undocumented) - search(url: string, options?: SearchOptions): Promise; - // (undocumented) - toString(): string; + constructor( + integration: GitLabIntegration, + deps: { + treeResponseFactory: ReadTreeResponseFactory; + }, + ); + // (undocumented) + static factory: ReaderFactory; + // (undocumented) + read(url: string): Promise; + // (undocumented) + readTree(url: string, options?: ReadTreeOptions): Promise; + // (undocumented) + search(url: string, options?: SearchOptions): Promise; + // (undocumented) + toString(): string; } // @public @@ -254,32 +299,32 @@ export function notFoundHandler(): RequestHandler; // @public export type PluginCacheManager = { - getClient: (options?: ClientOptions) => CacheClient; + getClient: (options?: ClientOptions) => CacheClient; }; // @public export interface PluginDatabaseManager { - getClient(): Promise; + getClient(): Promise; } // @public export type PluginEndpointDiscovery = { - getBaseUrl(pluginId: string): Promise; - getExternalBaseUrl(pluginId: string): Promise; + getBaseUrl(pluginId: string): Promise; + getExternalBaseUrl(pluginId: string): Promise; }; // @public (undocumented) export type ReadTreeResponse = { - files(): Promise; - archive(): Promise; - dir(options?: ReadTreeResponseDirOptions): Promise; - etag: string; + files(): Promise; + archive(): Promise; + dir(options?: ReadTreeResponseDirOptions): Promise; + etag: string; }; // @public export type ReadTreeResponseFile = { - path: string; - content(): Promise; + path: string; + content(): Promise; }; // @public @@ -290,37 +335,37 @@ export function resolvePackagePath(name: string, ...paths: string[]): string; // @public (undocumented) export type RunContainerOptions = { - imageName: string; - command?: string | string[]; - args: string[]; - logStream?: Writable; - mountDirs?: Record; - workingDir?: string; - envVars?: Record; + imageName: string; + command?: string | string[]; + args: string[]; + logStream?: Writable; + mountDirs?: Record; + workingDir?: string; + envVars?: Record; }; // @public export type SearchResponse = { - files: SearchResponseFile[]; - etag: string; + files: SearchResponseFile[]; + etag: string; }; // @public export type SearchResponseFile = { - url: string; - content(): Promise; + url: string; + content(): Promise; }; // @public (undocumented) export type ServiceBuilder = { - loadConfig(config: ConfigReader): ServiceBuilder; - setPort(port: number): ServiceBuilder; - setHost(host: string): ServiceBuilder; - setLogger(logger: Logger): ServiceBuilder; - enableCors(options: cors.CorsOptions): ServiceBuilder; - setHttpsSettings(settings: HttpsSettings): ServiceBuilder; - addRouter(root: string, router: Router | RequestHandler): ServiceBuilder; - start(): Promise; + loadConfig(config: ConfigReader): ServiceBuilder; + setPort(port: number): ServiceBuilder; + setHost(host: string): ServiceBuilder; + setLogger(logger: Logger): ServiceBuilder; + enableCors(options: cors.CorsOptions): ServiceBuilder; + setHttpsSettings(settings: HttpsSettings): ServiceBuilder; + addRouter(root: string, router: Router | RequestHandler): ServiceBuilder; + start(): Promise; }; // @public (undocumented) @@ -328,52 +373,58 @@ export function setRootLogger(newLogger: winston.Logger): void; // @public export class SingleConnectionDatabaseManager { - forPlugin(pluginId: string): PluginDatabaseManager; - static fromConfig(config: Config): SingleConnectionDatabaseManager; - } + forPlugin(pluginId: string): PluginDatabaseManager; + static fromConfig(config: Config): SingleConnectionDatabaseManager; +} // @public export class SingleHostDiscovery implements PluginEndpointDiscovery { - static fromConfig(config: Config, options?: { - basePath?: string; - }): SingleHostDiscovery; - // (undocumented) - getBaseUrl(pluginId: string): Promise; - // (undocumented) - getExternalBaseUrl(pluginId: string): Promise; - } + static fromConfig( + config: Config, + options?: { + basePath?: string; + }, + ): SingleHostDiscovery; + // (undocumented) + getBaseUrl(pluginId: string): Promise; + // (undocumented) + getExternalBaseUrl(pluginId: string): Promise; +} // @public (undocumented) export type StatusCheck = () => Promise; // @public -export function statusCheckHandler(options?: StatusCheckHandlerOptions): Promise; +export function statusCheckHandler( + options?: StatusCheckHandlerOptions, +): Promise; // @public (undocumented) export interface StatusCheckHandlerOptions { - statusCheck?: StatusCheck; + statusCheck?: StatusCheck; } // @public export type UrlReader = { - read(url: string): Promise; - readTree(url: string, options?: ReadTreeOptions): Promise; - search(url: string, options?: SearchOptions): Promise; + read(url: string): Promise; + readTree(url: string, options?: ReadTreeOptions): Promise; + search(url: string, options?: SearchOptions): Promise; }; // @public export class UrlReaders { - static create({ logger, config, factories }: CreateOptions): UrlReader; - static default({ logger, config, factories }: CreateOptions): UrlReader; + static create({ logger, config, factories }: CreateOptions): UrlReader; + static default({ logger, config, factories }: CreateOptions): UrlReader; } // @public -export function useHotCleanup(_module: NodeModule, cancelEffect: () => void): void; +export function useHotCleanup( + _module: NodeModule, + cancelEffect: () => void, +): void; // @public export function useHotMemoize(_module: NodeModule, valueFactory: () => T): T; - // (No @packageDocumentation comment for this package) - ``` diff --git a/packages/backend-common/config.d.ts b/packages/backend-common/config.d.ts index 875660a594..d735f39935 100644 --- a/packages/backend-common/config.d.ts +++ b/packages/backend-common/config.d.ts @@ -14,6 +14,23 @@ * limitations under the License. */ +export type PluginDatabaseConfig = + | { + /** Database client to use for plugin. */ + client?: 'sqlite3'; + /** Database connection to use with plugin. */ + connection?: ':memory:' | string | { filename: string }; + } + | { + /** Database client to use for plugin. */ + client?: 'pg'; + /** + * PostgreSQL connection string or knex configuration object for plugin. + * @secret + */ + connection?: string | object; + }; + export interface Config { app: { baseUrl: string; // defined in core, but repeated here without doc @@ -58,6 +75,12 @@ export interface Config { | { client: 'sqlite3'; connection: ':memory:' | string | { filename: string }; + /** Optional sqlite3 database filename prefix. */ + prefix?: string; + /** Override database config per plugin. */ + plugin?: { + [pluginId: string]: PluginDatabaseConfig; + }; } | { client: 'pg'; @@ -66,6 +89,12 @@ export interface Config { * @secret */ connection: string | object; + /** Optional PostgreSQL database prefix. */ + prefix?: string; + /** Override database config per plugin. */ + plugin?: { + [pluginId: string]: PluginDatabaseConfig; + }; }; /** Cache connection configuration, select cache type using the `store` field */ diff --git a/packages/backend-common/src/database/PluginConnection.test.ts b/packages/backend-common/src/database/PluginConnection.test.ts new file mode 100644 index 0000000000..e3a5157d7d --- /dev/null +++ b/packages/backend-common/src/database/PluginConnection.test.ts @@ -0,0 +1,325 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { ConfigReader } from '@backstage/config'; +import { omit } from 'lodash'; +import { createDatabaseClient, ensureDatabaseExists } from './connection'; +import { PluginConnectionDatabaseManager } from './PluginConnection'; + +jest.mock('./connection', () => ({ + ...jest.requireActual('./connection'), + createDatabaseClient: jest.fn(), + ensureDatabaseExists: jest.fn(), +})); + +describe('PluginConnectionDatabaseManager', () => { + // This is similar to the ts-jest `mocked` helper. + const mocked = (f: Function) => f as jest.Mock; + + afterEach(() => jest.resetAllMocks()); + + describe('PluginConnectionDatabaseManager.fromConfig', () => { + const backendConfig = { + backend: { + database: { + client: 'pg', + connection: { + host: 'localhost', + user: 'foo', + password: 'bar', + database: 'foodb', + }, + }, + }, + }; + const defaultConfig = () => new ConfigReader(backendConfig); + + it('accesses the backend.database key', () => { + const getConfig = jest.fn(); + const config = defaultConfig(); + config.getConfig = getConfig; + + PluginConnectionDatabaseManager.fromConfig(config); + + expect(getConfig.mock.calls[0][0]).toEqual('backend.database'); + }); + }); + + describe('PluginConnectionDatabaseManager.forPlugin', () => { + const config = { + backend: { + database: { + client: 'pg', + prefix: 'test_prefix_', + connection: { + host: 'localhost', + user: 'foo', + password: 'bar', + database: 'foodb', + }, + plugin: { + testdbname: { + connection: { + database: 'database_name_overriden', + }, + }, + differentclient: { + client: 'sqlite3', + connection: { + filename: 'plugin_with_different_client', + }, + }, + differentclientconnstring: { + client: 'sqlite3', + connection: ':inmemory:', + }, + stringoverride: { + connection: 'postgresql://testuser:testpass@acme:5432/userdbname', + }, + }, + }, + }, + }; + const manager = PluginConnectionDatabaseManager.fromConfig( + new ConfigReader(config), + ); + + it('connects to a plugin database using default config', async () => { + const pluginId = 'pluginwithoutconfig'; + + await manager.forPlugin(pluginId).getClient(); + expect(mocked(createDatabaseClient)).toHaveBeenCalledTimes(1); + + const mockCalls = mocked(createDatabaseClient).mock.calls.splice(-1); + const [baseConfig, overrides] = mockCalls[0]; + + // default config should be passed through to underlying connector + expect(baseConfig.get()).toMatchObject({ + client: 'pg', + connection: omit(config.backend.database.connection, ['database']), + }); + + // override using database name generated from pluginId and prefix + expect(overrides).toMatchObject({ + connection: { + database: `${config.backend.database.prefix}${pluginId}`, + }, + }); + }); + + it('provides a plugin db which uses components from top level connection string', async () => { + const testManager = PluginConnectionDatabaseManager.fromConfig( + new ConfigReader({ + backend: { + database: { + client: 'pg', + connection: 'postgresql://foo:bar@acme:5432/foodb', + }, + }, + }), + ); + + await testManager.forPlugin('pluginwithoutconfig').getClient(); + const mockCalls = mocked(createDatabaseClient).mock.calls.splice(-1); + const [baseConfig, overrides] = mockCalls[0]; + + // parsed connection string **without** db name should be passed through + expect(baseConfig.get()).toMatchObject({ + connection: { + host: 'acme', + user: 'foo', + password: 'bar', + port: '5432', + }, + }); + + // we expect a pg database name override with ${prefix} followed by pluginId + expect(overrides).toHaveProperty( + 'connection.database', + expect.stringContaining('pluginwithoutconfig'), + ); + }); + + it('uses top level sqlite database filename if plugin config is not present', async () => { + const testManager = PluginConnectionDatabaseManager.fromConfig( + new ConfigReader({ + backend: { + database: { + client: 'sqlite3', + connection: 'some-file-path', + }, + }, + }), + ); + + await testManager.forPlugin('pluginwithoutconfig').getClient(); + const mockCalls = mocked(createDatabaseClient).mock.calls.splice(-1); + const [_, overrides] = mockCalls[0]; + + expect(overrides).toHaveProperty( + 'connection.filename', + expect.stringContaining('some-file-path'), + ); + }); + + it('provides an inmemory sqlite database if top level is also inmemory and plugin config is not present', async () => { + const testManager = PluginConnectionDatabaseManager.fromConfig( + new ConfigReader({ + backend: { + database: { + client: 'sqlite3', + connection: ':inmemory:', + }, + }, + }), + ); + + await testManager.forPlugin('pluginwithoutconfig').getClient(); + const mockCalls = mocked(createDatabaseClient).mock.calls.splice(-1); + const [_, overrides] = mockCalls[0]; + + expect(overrides).toHaveProperty( + 'connection.filename', + expect.stringContaining(':inmemory:'), + ); + }); + + it('connects to a plugin database using a specific database name', async () => { + // testdbname.connection.database is set in config + await manager.forPlugin('testdbname').getClient(); + + const mockCalls = mocked(createDatabaseClient).mock.calls.splice(-1); + const [_baseConfig, overrides] = mockCalls[0]; + + // simple case where only database name is overriden + expect(overrides).toMatchObject({ + connection: { + database: 'database_name_overriden', + }, + }); + }); + + it('ensure plugin specific database is created', async () => { + const pluginId = 'testdbname'; + // testdbname.connection.database is set in config + await manager.forPlugin(pluginId).getClient(); + + const mockCalls = mocked(ensureDatabaseExists).mock.calls.splice(-1); + const [_, dbname] = mockCalls[0]; + + expect(dbname).toEqual( + config.backend.database.plugin[pluginId].connection.database, + ); + }); + + it('provides different plugins with their own databases', async () => { + await manager.forPlugin('plugin1').getClient(); + await manager.forPlugin('plugin2').getClient(); + + expect(mocked(createDatabaseClient)).toHaveBeenCalledTimes(2); + + const mockCalls = mocked(createDatabaseClient).mock.calls; + const [plugin1CallArgs, plugin2CallArgs] = mockCalls; + + // database name overrides should be different + expect(plugin1CallArgs[1].connection.database).not.toEqual( + plugin2CallArgs[1].connection.database, + ); + }); + + it('uses plugin connection as base if default client is different from plugin client', async () => { + const pluginId = 'differentclient'; + await manager.forPlugin(pluginId).getClient(); + + const mockCalls = mocked(createDatabaseClient).mock.calls.splice(-1); + const [baseConfig, _overrides] = mockCalls[0]; + + // plugin connection should be used as base config, client is different + expect(baseConfig.get()).toMatchObject({ + client: 'sqlite3', + connection: config.backend.database.plugin[pluginId].connection, + }); + }); + + it('provides database client specific base and override when client set under plugin', async () => { + const pluginId = 'differentclient'; + await manager.forPlugin(pluginId).getClient(); + + const mockCalls = mocked(createDatabaseClient).mock.calls.splice(-1); + const [baseConfig, overrides] = mockCalls[0]; + + // plugin client should be sqlite3 + expect(baseConfig.get().client).toEqual('sqlite3'); + + // sqlite3 uses 'filename' instead of 'database' + expect(overrides).toHaveProperty('connection.filename'); + }); + + it('provides database client specific base from plugin connection string when client set under plugin', async () => { + const pluginId = 'differentclientconnstring'; + await manager.forPlugin(pluginId).getClient(); + + const mockCalls = mocked(createDatabaseClient).mock.calls.splice(-1); + const [baseConfig, overrides] = mockCalls[0]; + + expect(baseConfig.get().client).toEqual('sqlite3'); + + expect(overrides).toHaveProperty('connection.filename', ':inmemory:'); + }); + + it('generates a database name override when prefix is not explicitly set', async () => { + const testManager = PluginConnectionDatabaseManager.fromConfig( + new ConfigReader({ + backend: { + database: { + client: 'pg', + connection: { + host: 'localhost', + user: 'foo', + password: 'bar', + database: 'foodb', + }, + }, + }, + }), + ); + + await testManager.forPlugin('testplugin').getClient(); + const mockCalls = mocked(createDatabaseClient).mock.calls.splice(-1); + const [_baseConfig, overrides] = mockCalls[0]; + + expect(overrides).toHaveProperty( + 'connection.database', + expect.stringContaining(PluginConnectionDatabaseManager.DEFAULT_PREFIX), + ); + }); + + it('uses values from plugin connection string if top level client should be used', async () => { + const pluginId = 'stringoverride'; + await manager.forPlugin(pluginId).getClient(); + + const mockCalls = mocked(createDatabaseClient).mock.calls.splice(-1); + const [baseConfig, overrides] = mockCalls[0]; + + // plugin client should be pg + expect(baseConfig.get().client).toEqual('pg'); + + expect(overrides).toHaveProperty( + 'connection.database', + expect.stringContaining('userdbname'), + ); + }); + }); +}); diff --git a/packages/backend-common/src/database/PluginConnection.ts b/packages/backend-common/src/database/PluginConnection.ts new file mode 100644 index 0000000000..18c209c8d5 --- /dev/null +++ b/packages/backend-common/src/database/PluginConnection.ts @@ -0,0 +1,160 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Knex } from 'knex'; +import { omit } from 'lodash'; +import { Config, ConfigReader } from '@backstage/config'; +import { + createDatabaseClient, + ensureDatabaseExists, + createNameOverride, + normalizeConnection, +} from './connection'; +import { PluginDatabaseManager } from './types'; + +function pluginPath(pluginId: string): string { + return `plugin.${pluginId}`; +} + +export class PluginConnectionDatabaseManager { + static readonly DEFAULT_PREFIX = 'backstage_plugin_'; + + /** + * Creates a PluginConnectionDatabaseManager from `backend.database` config. + * + * The database manager allows the user to set connection and client settings on a per pluginId + * basis by defining a database config block under `plugin.` in addition to top level + * defaults. Optionally, a user may set `prefix` which is used to prefix generated database + * names if config is not provided. + * + * @param config The loaded application configuration. + */ + static fromConfig(config: Config): PluginConnectionDatabaseManager { + return new PluginConnectionDatabaseManager( + config.getConfig('backend.database'), + ); + } + + private constructor(private readonly config: Config) {} + + /** + * Generates a PluginDatabaseManager for consumption by plugins. + * + * @param pluginId The plugin that the database manager should be created for. Plugin names should be unique + * as they are used to look up database config overrides under `backend.database.plugin`. + */ + forPlugin(pluginId: string): PluginDatabaseManager { + const _this = this; + + return { + getClient(): Promise { + return _this.getDatabase(pluginId); + }, + }; + } + + /** + * Provides the canonical database name for a given pluginId. + * + * This method provides the effective database name which is determined using global + * and plugin specific database config. If no explicit database name is configured, + * this method will provide a generated name which is the pluginId prefixed using + * the value from `PluginConnectionDatabaseManager.DEFAULT_PREFIX`. + * + * @param pluginId Lookup the database name for given plugin + * */ + getDatabaseName(pluginId: string): string { + const pluginConfig: Config = this.getConfigForPlugin(pluginId); + + // determine root sqlite config to pass through as this is a special case + const rootConnection = this.config.get('connection'); + const rootSqliteName = + typeof rootConnection === 'string' + ? rootConnection + : this.config.getOptionalString('connection.filename') ?? ':inmemory:'; + + const prefix = + this.config.getOptionalString('prefix') ?? + PluginConnectionDatabaseManager.DEFAULT_PREFIX; + + const isSqlite = this.config.getString('client') === 'sqlite3'; + return ( + // attempt to lookup pg and mysql database name + pluginConfig.getOptionalString('connection.database') ?? + // attempt to lookup sqlite3 database file name + pluginConfig.getOptionalString('connection.filename') ?? + // if root is sqlite - attempt to use top level connection, fallback to :inmemory: + (isSqlite ? rootSqliteName : null) ?? + // generate a database name using prefix and pluginId + `${prefix}${pluginId}` + ); + } + + /** + * Provides a base database connector config by merging different config sources. + * + * This method provides a baseConfig for a database connector without the target + * database's name property ('database', 'filename'). The client type is determined + * by plugin specific config which uses the default as the fallback. + * + * If the client type is the same as the plugin or not specified, the global + * connection config will be extended with plugin specific config. + * + * @param pluginId The plugin that the database baseConfig should correspond to + * */ + private getConfigForPlugin(pluginId: string): Config { + const pluginConfig = this.config.getOptionalConfig(pluginPath(pluginId)); + + const baseClient = this.config.getString('client'); + const client = pluginConfig?.getOptionalString('client') ?? baseClient; + + const baseConnection = normalizeConnection( + this.config.get('connection'), + baseClient, + ); + const connection = normalizeConnection( + pluginConfig?.getOptional('connection') ?? {}, + client, + ); + + return new ConfigReader({ + client, + connection: { + // if same client type, extend original connection config without dbname config + ...(client === baseClient + ? omit(baseConnection, ['database', 'filename']) + : {}), + ...connection, + }, + }); + } + + private async getDatabase(pluginId: string): Promise { + const pluginConfig = this.getConfigForPlugin(pluginId); + + await ensureDatabaseExists(pluginConfig, this.getDatabaseName(pluginId)); + return createDatabaseClient( + pluginConfig, + this.getDatabaseOverrides(pluginId), + ); + } + + private getDatabaseOverrides(pluginId: string): Knex.Config { + return createNameOverride( + this.getConfigForPlugin(pluginId).get('client'), + this.getDatabaseName(pluginId), + ); + } +} diff --git a/packages/backend-common/src/database/connection.test.ts b/packages/backend-common/src/database/connection.test.ts index 6ab163ff7f..fa7ccd480a 100644 --- a/packages/backend-common/src/database/connection.test.ts +++ b/packages/backend-common/src/database/connection.test.ts @@ -15,7 +15,11 @@ */ import { ConfigReader } from '@backstage/config'; -import { createDatabaseClient } from './connection'; +import { + createDatabaseClient, + createNameOverride, + parseConnectionString, +} from './connection'; describe('database connection', () => { describe('createDatabaseClient', () => { @@ -103,4 +107,49 @@ describe('database connection', () => { ).toThrowError(); }); }); + + describe('createNameOverride', () => { + it('returns Knex config for postgres', () => { + expect(createNameOverride('pg', 'testpg')).toHaveProperty( + 'connection.database', + 'testpg', + ); + }); + + it('returns Knex config for sqlite', () => { + expect(createNameOverride('sqlite3', 'testsqlite')).toHaveProperty( + 'connection.filename', + 'testsqlite', + ); + }); + + it('returns Knex config for mysql', () => { + expect(createNameOverride('mysql', 'testmysql')).toHaveProperty( + 'connection.database', + 'testmysql', + ); + }); + + it('throws an error for unknown connection', () => { + expect(() => createNameOverride('unknown', 'testname')).toThrowError(); + }); + }); + + describe('parseConnectionString', () => { + it('returns parsed Knex.StaticConnectionConfig for postgres', () => { + expect( + parseConnectionString('postgresql://foo:bar@acme:5432/foodb', 'pg'), + ).toHaveProperty('database', 'foodb'); + }); + + it('returns parsed Knex.StaticConnectionConfig for mysql2', () => { + expect( + parseConnectionString('mysql://foo:bar@acme:3306/foodb', 'mysql2'), + ).toHaveProperty('database', 'foodb'); + }); + + it('throws an error if client hint is not provided', () => { + expect(() => parseConnectionString('sqlite://')).toThrow(); + }); + }); }); diff --git a/packages/backend-common/src/database/connection.ts b/packages/backend-common/src/database/connection.ts index fb16915cf4..4a4a8c6ba2 100644 --- a/packages/backend-common/src/database/connection.ts +++ b/packages/backend-common/src/database/connection.ts @@ -14,14 +14,30 @@ * limitations under the License. */ -import { Config } from '@backstage/config'; +import { Config, JsonObject } from '@backstage/config'; +import { InputError } from '@backstage/errors'; import knexFactory, { Knex } from 'knex'; import { mergeDatabaseConfig } from './config'; -import { createMysqlDatabaseClient, ensureMysqlDatabaseExists } from './mysql'; -import { createPgDatabaseClient, ensurePgDatabaseExists } from './postgres'; -import { createSqliteDatabaseClient } from './sqlite3'; +import { DatabaseConnector } from './connector'; -type DatabaseClient = 'pg' | 'sqlite3' | string; +import { mysqlConnector } from './mysql'; +import { pgConnector } from './postgres'; +import { sqlite3Connector } from './sqlite3'; + +type DatabaseClient = 'pg' | 'sqlite3' | 'mysql' | 'mysql2' | string; + +/** + * Mapping of client type to supported database connectors + * + * Database connectors can be aliased here, for example mysql2 uses + * the same connector as mysql. + * */ +const ConnectorMapping: Record = { + pg: pgConnector, + sqlite3: sqlite3Connector, + mysql: mysqlConnector, + mysql2: mysqlConnector, +}; /** * Creates a knex database connection @@ -35,15 +51,10 @@ export function createDatabaseClient( ) { const client: DatabaseClient = dbConfig.getString('client'); - if (client === 'pg') { - return createPgDatabaseClient(dbConfig, overrides); - } else if (client === 'mysql' || client === 'mysql2') { - return createMysqlDatabaseClient(dbConfig, overrides); - } else if (client === 'sqlite3') { - return createSqliteDatabaseClient(dbConfig, overrides); - } - - return knexFactory(mergeDatabaseConfig(dbConfig.get(), overrides)); + return ( + ConnectorMapping[client]?.createClient(dbConfig, overrides) ?? + knexFactory(mergeDatabaseConfig(dbConfig.get(), overrides)) + ); } /** @@ -61,11 +72,63 @@ export async function ensureDatabaseExists( ) { const client: DatabaseClient = dbConfig.getString('client'); - if (client === 'pg') { - return ensurePgDatabaseExists(dbConfig, ...databases); - } else if (client === 'mysql' || client === 'mysql2') { - return ensureMysqlDatabaseExists(dbConfig, ...databases); + return ConnectorMapping[client]?.ensureDatabaseExists?.( + dbConfig, + ...databases, + ); +} + +/** + * Provides a Knex.Config object with the provided database name for a given client. + * */ +export function createNameOverride( + client: string, + name: string, +): Partial { + try { + return ConnectorMapping[client].createNameOverride(name); + } catch (e) { + throw new InputError( + `Unable to create database name override for '${client}' connector`, + e, + ); + } +} + +/** + * Parses a connection string for a given client and provides a connection config. + * */ +export function parseConnectionString( + connectionString: string, + client?: string, +): Knex.StaticConnectionConfig { + if (typeof client === 'undefined' || client === null) { + throw new InputError( + 'Database connection string client type auto-detection is not yet supported.', + ); } - return undefined; + try { + return ConnectorMapping[client].parseConnectionString(connectionString); + } catch (e) { + throw new InputError( + `Unable to parse connection string for '${client}' connector`, + ); + } +} + +/** + * Normalizes a connection config or string into an object which can be passed to Knex. + * */ +export function normalizeConnection( + connection: Knex.StaticConnectionConfig | JsonObject | string, + client: string, +): Record { + if (typeof connection === 'undefined' || connection === null) { + return {}; + } + + return typeof connection === 'string' || connection instanceof String + ? parseConnectionString(connection as string, client) + : connection; } diff --git a/packages/backend-common/src/database/connector.ts b/packages/backend-common/src/database/connector.ts new file mode 100644 index 0000000000..0336510f19 --- /dev/null +++ b/packages/backend-common/src/database/connector.ts @@ -0,0 +1,30 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Config } from '@backstage/config'; +import { Knex } from 'knex'; + +export interface DatabaseConnector { + createClient(dbConfig: Config, overrides?: Partial): Knex; + createNameOverride(name: string): Partial; + parseConnectionString( + connectionString: string, + client?: string, + ): Knex.StaticConnectionConfig; + ensureDatabaseExists?( + dbConfig: Config, + ...databases: Array + ): Promise; +} diff --git a/packages/backend-common/src/database/index.ts b/packages/backend-common/src/database/index.ts index c81153aa62..7a8a81e187 100644 --- a/packages/backend-common/src/database/index.ts +++ b/packages/backend-common/src/database/index.ts @@ -17,3 +17,4 @@ export * from './connection'; export * from './types'; export * from './SingleConnection'; +export * from './PluginConnection'; diff --git a/packages/backend-common/src/database/mysql.ts b/packages/backend-common/src/database/mysql.ts index be4632baf6..3bd874de11 100644 --- a/packages/backend-common/src/database/mysql.ts +++ b/packages/backend-common/src/database/mysql.ts @@ -18,6 +18,7 @@ import { Config } from '@backstage/config'; import { InputError } from '@backstage/errors'; import knexFactory, { Knex } from 'knex'; import { mergeDatabaseConfig } from './config'; +import { DatabaseConnector } from './connector'; import yn from 'yn'; /** @@ -159,3 +160,23 @@ export async function ensureMysqlDatabaseExists( await admin.destroy(); } } + +export function createMysqlNameOverride(name: string): Partial { + return { + connection: { + database: name, + }, + }; +} + +/** + * MySql database connector. + * + * Exposes database connector functionality via an immutable object. + * */ +export const mysqlConnector: DatabaseConnector = Object.freeze({ + createClient: createMysqlDatabaseClient, + ensureDatabaseExists: ensureMysqlDatabaseExists, + createNameOverride: createMysqlNameOverride, + parseConnectionString: parseMysqlConnectionString, +}); diff --git a/packages/backend-common/src/database/postgres.ts b/packages/backend-common/src/database/postgres.ts index e05eab86e5..40c6a41504 100644 --- a/packages/backend-common/src/database/postgres.ts +++ b/packages/backend-common/src/database/postgres.ts @@ -17,6 +17,7 @@ import knexFactory, { Knex } from 'knex'; import { Config } from '@backstage/config'; import { mergeDatabaseConfig } from './config'; +import { DatabaseConnector } from './connector'; /** * Creates a knex postgres database connection @@ -131,3 +132,23 @@ export async function ensurePgDatabaseExists( await admin.destroy(); } } + +export function createPgNameOverride(name: string): Partial { + return { + connection: { + database: name, + }, + }; +} + +/** + * PostgreSQL database connector. + * + * Exposes database connector functionality via an immutable object. + * */ +export const pgConnector: DatabaseConnector = Object.freeze({ + createClient: createPgDatabaseClient, + ensureDatabaseExists: ensurePgDatabaseExists, + createNameOverride: createPgNameOverride, + parseConnectionString: parsePgConnectionString, +}); diff --git a/packages/backend-common/src/database/sqlite3.ts b/packages/backend-common/src/database/sqlite3.ts index d4169e3899..98c39b9c97 100644 --- a/packages/backend-common/src/database/sqlite3.ts +++ b/packages/backend-common/src/database/sqlite3.ts @@ -19,6 +19,7 @@ import { ensureDirSync } from 'fs-extra'; import knexFactory, { Knex } from 'knex'; import path from 'path'; import { mergeDatabaseConfig } from './config'; +import { DatabaseConnector } from './connector'; /** * Creates a knex sqlite3 database connection @@ -99,3 +100,28 @@ export function buildSqliteDatabaseConfig( return config; } + +export function createSqliteNameOverride(name: string): Partial { + return { + connection: parseSqliteConnectionString(name), + }; +} + +export function parseSqliteConnectionString( + name: string, +): Knex.Sqlite3ConnectionConfig { + return { + filename: name, + }; +} + +/** + * Sqlite3 database connector. + * + * Exposes database connector functionality via an immutable object. + * */ +export const sqlite3Connector: DatabaseConnector = Object.freeze({ + createClient: createSqliteDatabaseClient, + createNameOverride: createSqliteNameOverride, + parseConnectionString: parseSqliteConnectionString, +}); diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index d43c8f0101..9cab64e797 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -29,7 +29,7 @@ import { getRootLogger, loadBackendConfig, notFoundHandler, - SingleConnectionDatabaseManager, + PluginConnectionDatabaseManager, SingleHostDiscovery, UrlReaders, useHotMemoize, @@ -59,7 +59,7 @@ function makeCreateEnv(config: Config) { root.info(`Created UrlReader ${reader}`); - const databaseManager = SingleConnectionDatabaseManager.fromConfig(config); + const databaseManager = PluginConnectionDatabaseManager.fromConfig(config); const cacheManager = CacheManager.fromConfig(config); return (plugin: string): PluginEnvironment => { From 4d17ececc53f76a9fe68991208523daf6ad42dc6 Mon Sep 17 00:00:00 2001 From: Minn Soe Date: Mon, 17 May 2021 19:38:57 +0100 Subject: [PATCH 13/93] docs: fix typo in db manager changeset and clarify Signed-off-by: Minn Soe --- .changeset/five-donkeys-brake.md | 2 +- .../tutorials/configuring-plugin-databases.md | 44 +++++++++++++------ 2 files changed, 31 insertions(+), 15 deletions(-) diff --git a/.changeset/five-donkeys-brake.md b/.changeset/five-donkeys-brake.md index c9db3112a9..15d2cf7381 100644 --- a/.changeset/five-donkeys-brake.md +++ b/.changeset/five-donkeys-brake.md @@ -27,7 +27,7 @@ backend: database: 'database_name_overriden' scaffolder: client: 'sqlite3' - connection: ':inmemory' + connection: ':inmemory:' ``` Existing backstage installations can be migrated by swapping out the database diff --git a/docs/tutorials/configuring-plugin-databases.md b/docs/tutorials/configuring-plugin-databases.md index b6281cb0ae..b65addd7a8 100644 --- a/docs/tutorials/configuring-plugin-databases.md +++ b/docs/tutorials/configuring-plugin-databases.md @@ -9,12 +9,19 @@ There are occasions where it may be difficult to deploy Backstage with automatically created databases in production due to access control or other restrictions. For example, your infrastructure might be defined as code using tools such as Terraform or AWS CloudFormation where the name of each database is -defined, created and assigned explicitly. +defined, created and assigned explicitly. You may also need to use different +credentials for each database or use a set of credentials without the +permissions needed to create databases. -`@backstage/backend-common` provides an alternate database manager which allows -you to set the client and database connection on a per plugin basis. This means -that you can do selectively run certain plugins in memory with `sqlite3`, set -different connection config including the name of the database and more. +`@backstage/backend-common` provides an alternate database manager, +`PluginConnectionDatabaseManager`, which allows the developer to set the client +and database connection on a per plugin basis in addition to the default client +and connection configuration. This means that you can use a `sqlite3` in memory +database for a specific plugin whilst using `postgres` for everything else and +so on. + +The database manager also allows you to change the database name prefix which is +used when a plugin database isn't explicitly configured. There are two additional configuration options for this database manager: @@ -41,12 +48,20 @@ yarn add pg yarn add sqlite3 ``` +From an operational perspective, you only need to install drivers for clients +that are actively used. + ## Add Configuration -To override the default prefix, `backstage_plugin_`, set -`backend.database.prefix` as shown below. This will use databases such as -`my_company_catalog` and `my_company_auth` instead of `backstage_plugin_catalog` -and `backstage_plugin_auth`. +You can set the same type of values for `backend.database..client` and +`backend.database..connection` which are also accepted at the top +level. + +It is possible to override the default database name prefix, +`backstage_plugin_`, which is used when a name isn't explicitly defined. Set +`backend.database.prefix` as shown below. The database names for plugins such as +`catalog` and `auth` would now be `my_company_catalog` and `my_company_auth` +instead of `backstage_plugin_catalog` and `backstage_plugin_auth`. ```yaml backend: @@ -94,11 +109,12 @@ function makeCreateEnv(config: Config) { The `PluginConnectionDatabaseManager` preserves the behaviour of the `SingleConnectionDatabaseManager`. If the database does not exist, it will -attempt to create it. You should ensure the databases that you configure exists -and that the connection details have the appropriate permissions to work with -each of the given databases if you are using this database manager to set the -database name upfront. If each database needs its own connection username, -password or host - you may set them under the plugin's `connection` block. +attempt to create it. + +If you are using this database manager to set the database name upfront because +the credentials do not have permissions to create databases, you must ensure +they exist before starting the service. The service will not be able to create +them, it can only use them. `sqlite3` databases do not need to be created upfront as with the existing database manager. From 630b75798a93f95b2f5fcf87e156b5d5c7d31ff4 Mon Sep 17 00:00:00 2001 From: Minn Soe Date: Mon, 17 May 2021 22:37:44 +0100 Subject: [PATCH 14/93] fix: return type of ensureDatabaseExists and run api-report Signed-off-by: Minn Soe --- packages/backend-common/api-report.md | 476 ++++++++---------- .../backend-common/src/database/connection.ts | 5 +- 2 files changed, 223 insertions(+), 258 deletions(-) diff --git a/packages/backend-common/api-report.md b/packages/backend-common/api-report.md index 2648e7af13..002cfaac89 100644 --- a/packages/backend-common/api-report.md +++ b/packages/backend-common/api-report.md @@ -3,6 +3,7 @@ > Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). ```ts + import { AzureIntegration } from '@backstage/integration'; import { BitbucketIntegration } from '@backstage/integration'; import { Config } from '@backstage/config'; @@ -15,6 +16,7 @@ import { GithubCredentialsProvider } from '@backstage/integration'; import { GitHubIntegration } from '@backstage/integration'; import { GitLabIntegration } from '@backstage/integration'; import * as http from 'http'; +import { JsonObject } from '@backstage/config'; import { JsonValue } from '@backstage/config'; import { Knex } from 'knex'; import { Logger } from 'winston'; @@ -30,55 +32,49 @@ import { Writable } from 'stream'; // @public (undocumented) export class AzureUrlReader implements UrlReader { - constructor( - integration: AzureIntegration, - deps: { - treeResponseFactory: ReadTreeResponseFactory; - }, - ); - // (undocumented) - static factory: ReaderFactory; - // (undocumented) - read(url: string): Promise; - // (undocumented) - readTree(url: string, options?: ReadTreeOptions): Promise; - // (undocumented) - search(url: string, options?: SearchOptions): Promise; - // (undocumented) - toString(): string; + constructor(integration: AzureIntegration, deps: { + treeResponseFactory: ReadTreeResponseFactory; + }); + // (undocumented) + static factory: ReaderFactory; + // (undocumented) + read(url: string): Promise; + // (undocumented) + readTree(url: string, options?: ReadTreeOptions): Promise; + // (undocumented) + search(url: string, options?: SearchOptions): Promise; + // (undocumented) + toString(): string; } // @public export class BitbucketUrlReader implements UrlReader { - constructor( - integration: BitbucketIntegration, - deps: { - treeResponseFactory: ReadTreeResponseFactory; - }, - ); - // (undocumented) - static factory: ReaderFactory; - // (undocumented) - read(url: string): Promise; - // (undocumented) - readTree(url: string, options?: ReadTreeOptions): Promise; - // (undocumented) - search(url: string, options?: SearchOptions): Promise; - // (undocumented) - toString(): string; + constructor(integration: BitbucketIntegration, deps: { + treeResponseFactory: ReadTreeResponseFactory; + }); + // (undocumented) + static factory: ReaderFactory; + // (undocumented) + read(url: string): Promise; + // (undocumented) + readTree(url: string, options?: ReadTreeOptions): Promise; + // (undocumented) + search(url: string, options?: SearchOptions): Promise; + // (undocumented) + toString(): string; } // @public export interface CacheClient { - delete(key: string): Promise; - get(key: string): Promise; - set(key: string, value: JsonValue, options?: CacheSetOptions): Promise; + delete(key: string): Promise; + get(key: string): Promise; + set(key: string, value: JsonValue, options?: CacheSetOptions): Promise; } // @public export class CacheManager { - forPlugin(pluginId: string): PluginCacheManager; - static fromConfig(config: Config, options?: CacheManagerOptions): CacheManager; + forPlugin(pluginId: string): PluginCacheManager; + static fromConfig(config: Config, options?: CacheManagerOptions): CacheManager; } // @public (undocumented) @@ -86,64 +82,48 @@ export const coloredFormat: winston.Logform.Format; // @public (undocumented) export interface ContainerRunner { - // (undocumented) - runContainer(opts: RunContainerOptions): Promise; + // (undocumented) + runContainer(opts: RunContainerOptions): Promise; } // @public @deprecated export const createDatabase: typeof createDatabaseClient; // @public -export function createDatabaseClient( - dbConfig: Config, - overrides?: Partial, -): Knex; +export function createDatabaseClient(dbConfig: Config, overrides?: Partial): Knex; + +// @public +export function createNameOverride(client: string, name: string): Partial; // @public (undocumented) -export function createRootLogger( - options?: winston.LoggerOptions, - env?: NodeJS.ProcessEnv, -): winston.Logger; +export function createRootLogger(options?: winston.LoggerOptions, env?: NodeJS.ProcessEnv): winston.Logger; // @public export function createServiceBuilder(_module: NodeModule): ServiceBuilderImpl; // @public (undocumented) -export function createStatusCheckRouter( - options: StatusCheckRouterOptions, -): Promise; +export function createStatusCheckRouter(options: StatusCheckRouterOptions): Promise; // @public (undocumented) export class DockerContainerRunner implements ContainerRunner { - constructor({ dockerClient }: { dockerClient: Docker }); - // (undocumented) - runContainer({ - imageName, - command, - args, - logStream, - mountDirs, - workingDir, - envVars, - }: RunContainerOptions): Promise; + constructor({ dockerClient }: { + dockerClient: Docker; + }); + // (undocumented) + runContainer({ imageName, command, args, logStream, mountDirs, workingDir, envVars, }: RunContainerOptions): Promise; } // @public -export function ensureDatabaseExists( - dbConfig: Config, - ...databases: Array -): Promise; +export function ensureDatabaseExists(dbConfig: Config, ...databases: Array): Promise; // @public -export function errorHandler( - options?: ErrorHandlerOptions, -): ErrorRequestHandler; +export function errorHandler(options?: ErrorHandlerOptions): ErrorRequestHandler; // @public (undocumented) export type ErrorHandlerOptions = { - showStackTraces?: boolean; - logger?: Logger; - logClientErrors?: boolean; + showStackTraces?: boolean; + logger?: Logger; + logClientErrors?: boolean; }; // @public (undocumented) @@ -154,177 +134,171 @@ export function getVoidLogger(): winston.Logger; // @public (undocumented) export class Git { - // (undocumented) - add({ dir, filepath }: { dir: string; filepath: string }): Promise; - // (undocumented) - addRemote({ - dir, - url, - remote, - }: { - dir: string; - remote: string; - url: string; - }): Promise; - // (undocumented) - clone({ - url, - dir, - ref, - }: { - url: string; - dir: string; - ref?: string; - }): Promise; - // (undocumented) - commit({ - dir, - message, - author, - committer, - }: { - dir: string; - message: string; - author: { - name: string; - email: string; - }; - committer: { - name: string; - email: string; - }; - }): Promise; - // (undocumented) - currentBranch({ - dir, - fullName, - }: { - dir: string; - fullName?: boolean; - }): Promise; - // (undocumented) - fetch({ dir, remote }: { dir: string; remote?: string }): Promise; - // (undocumented) - static fromAuth: ({ - username, - password, - logger, - }: { - username?: string | undefined; - password?: string | undefined; - logger?: Logger | undefined; - }) => Git; - // (undocumented) - init({ dir }: { dir: string }): Promise; - // (undocumented) - merge({ - dir, - theirs, - ours, - author, - committer, - }: { - dir: string; - theirs: string; - ours?: string; - author: { - name: string; - email: string; - }; - committer: { - name: string; - email: string; - }; - }): Promise; - // (undocumented) - push({ dir, remote }: { dir: string; remote: string }): Promise; - // (undocumented) - readCommit({ - dir, - sha, - }: { - dir: string; - sha: string; - }): Promise; - // (undocumented) - resolveRef({ dir, ref }: { dir: string; ref: string }): Promise; + // (undocumented) + add({ dir, filepath, }: { + dir: string; + filepath: string; + }): Promise; + // (undocumented) + addRemote({ dir, url, remote, }: { + dir: string; + remote: string; + url: string; + }): Promise; + // (undocumented) + clone({ url, dir, ref, }: { + url: string; + dir: string; + ref?: string; + }): Promise; + // (undocumented) + commit({ dir, message, author, committer, }: { + dir: string; + message: string; + author: { + name: string; + email: string; + }; + committer: { + name: string; + email: string; + }; + }): Promise; + // (undocumented) + currentBranch({ dir, fullName, }: { + dir: string; + fullName?: boolean; + }): Promise; + // (undocumented) + fetch({ dir, remote, }: { + dir: string; + remote?: string; + }): Promise; + // (undocumented) + static fromAuth: ({ username, password, logger, }: { + username?: string | undefined; + password?: string | undefined; + logger?: Logger | undefined; + }) => Git; + // (undocumented) + init({ dir }: { + dir: string; + }): Promise; + // (undocumented) + merge({ dir, theirs, ours, author, committer, }: { + dir: string; + theirs: string; + ours?: string; + author: { + name: string; + email: string; + }; + committer: { + name: string; + email: string; + }; + }): Promise; + // (undocumented) + push({ dir, remote }: { + dir: string; + remote: string; + }): Promise; + // (undocumented) + readCommit({ dir, sha, }: { + dir: string; + sha: string; + }): Promise; + // (undocumented) + resolveRef({ dir, ref, }: { + dir: string; + ref: string; + }): Promise; } // @public export class GithubUrlReader implements UrlReader { - constructor( - integration: GitHubIntegration, - deps: { - treeResponseFactory: ReadTreeResponseFactory; - credentialsProvider: GithubCredentialsProvider; - }, - ); - // (undocumented) - static factory: ReaderFactory; - // (undocumented) - read(url: string): Promise; - // (undocumented) - readTree(url: string, options?: ReadTreeOptions): Promise; - // (undocumented) - search(url: string, options?: SearchOptions): Promise; - // (undocumented) - toString(): string; + constructor(integration: GitHubIntegration, deps: { + treeResponseFactory: ReadTreeResponseFactory; + credentialsProvider: GithubCredentialsProvider; + }); + // (undocumented) + static factory: ReaderFactory; + // (undocumented) + read(url: string): Promise; + // (undocumented) + readTree(url: string, options?: ReadTreeOptions): Promise; + // (undocumented) + search(url: string, options?: SearchOptions): Promise; + // (undocumented) + toString(): string; } // @public (undocumented) export class GitlabUrlReader implements UrlReader { - constructor( - integration: GitLabIntegration, - deps: { - treeResponseFactory: ReadTreeResponseFactory; - }, - ); - // (undocumented) - static factory: ReaderFactory; - // (undocumented) - read(url: string): Promise; - // (undocumented) - readTree(url: string, options?: ReadTreeOptions): Promise; - // (undocumented) - search(url: string, options?: SearchOptions): Promise; - // (undocumented) - toString(): string; + constructor(integration: GitLabIntegration, deps: { + treeResponseFactory: ReadTreeResponseFactory; + }); + // (undocumented) + static factory: ReaderFactory; + // (undocumented) + read(url: string): Promise; + // (undocumented) + readTree(url: string, options?: ReadTreeOptions): Promise; + // (undocumented) + search(url: string, options?: SearchOptions): Promise; + // (undocumented) + toString(): string; } // @public export function loadBackendConfig(options: Options): Promise; +// @public +export function normalizeConnection(connection: Knex.StaticConnectionConfig | JsonObject | string, client: string): Record; + // @public export function notFoundHandler(): RequestHandler; +// @public +export function parseConnectionString(connectionString: string, client?: string): Knex.StaticConnectionConfig; + // @public export type PluginCacheManager = { - getClient: (options?: ClientOptions) => CacheClient; + getClient: (options?: ClientOptions) => CacheClient; }; +// @public (undocumented) +export class PluginConnectionDatabaseManager { + // (undocumented) + static readonly DEFAULT_PREFIX = "backstage_plugin_"; + forPlugin(pluginId: string): PluginDatabaseManager; + static fromConfig(config: Config): PluginConnectionDatabaseManager; + getDatabaseName(pluginId: string): string; + } + // @public export interface PluginDatabaseManager { - getClient(): Promise; + getClient(): Promise; } // @public export type PluginEndpointDiscovery = { - getBaseUrl(pluginId: string): Promise; - getExternalBaseUrl(pluginId: string): Promise; + getBaseUrl(pluginId: string): Promise; + getExternalBaseUrl(pluginId: string): Promise; }; // @public (undocumented) export type ReadTreeResponse = { - files(): Promise; - archive(): Promise; - dir(options?: ReadTreeResponseDirOptions): Promise; - etag: string; + files(): Promise; + archive(): Promise; + dir(options?: ReadTreeResponseDirOptions): Promise; + etag: string; }; // @public export type ReadTreeResponseFile = { - path: string; - content(): Promise; + path: string; + content(): Promise; }; // @public @@ -335,37 +309,37 @@ export function resolvePackagePath(name: string, ...paths: string[]): string; // @public (undocumented) export type RunContainerOptions = { - imageName: string; - command?: string | string[]; - args: string[]; - logStream?: Writable; - mountDirs?: Record; - workingDir?: string; - envVars?: Record; + imageName: string; + command?: string | string[]; + args: string[]; + logStream?: Writable; + mountDirs?: Record; + workingDir?: string; + envVars?: Record; }; // @public export type SearchResponse = { - files: SearchResponseFile[]; - etag: string; + files: SearchResponseFile[]; + etag: string; }; // @public export type SearchResponseFile = { - url: string; - content(): Promise; + url: string; + content(): Promise; }; // @public (undocumented) export type ServiceBuilder = { - loadConfig(config: ConfigReader): ServiceBuilder; - setPort(port: number): ServiceBuilder; - setHost(host: string): ServiceBuilder; - setLogger(logger: Logger): ServiceBuilder; - enableCors(options: cors.CorsOptions): ServiceBuilder; - setHttpsSettings(settings: HttpsSettings): ServiceBuilder; - addRouter(root: string, router: Router | RequestHandler): ServiceBuilder; - start(): Promise; + loadConfig(config: ConfigReader): ServiceBuilder; + setPort(port: number): ServiceBuilder; + setHost(host: string): ServiceBuilder; + setLogger(logger: Logger): ServiceBuilder; + enableCors(options: cors.CorsOptions): ServiceBuilder; + setHttpsSettings(settings: HttpsSettings): ServiceBuilder; + addRouter(root: string, router: Router | RequestHandler): ServiceBuilder; + start(): Promise; }; // @public (undocumented) @@ -373,58 +347,52 @@ export function setRootLogger(newLogger: winston.Logger): void; // @public export class SingleConnectionDatabaseManager { - forPlugin(pluginId: string): PluginDatabaseManager; - static fromConfig(config: Config): SingleConnectionDatabaseManager; -} + forPlugin(pluginId: string): PluginDatabaseManager; + static fromConfig(config: Config): SingleConnectionDatabaseManager; + } // @public export class SingleHostDiscovery implements PluginEndpointDiscovery { - static fromConfig( - config: Config, - options?: { - basePath?: string; - }, - ): SingleHostDiscovery; - // (undocumented) - getBaseUrl(pluginId: string): Promise; - // (undocumented) - getExternalBaseUrl(pluginId: string): Promise; -} + static fromConfig(config: Config, options?: { + basePath?: string; + }): SingleHostDiscovery; + // (undocumented) + getBaseUrl(pluginId: string): Promise; + // (undocumented) + getExternalBaseUrl(pluginId: string): Promise; + } // @public (undocumented) export type StatusCheck = () => Promise; // @public -export function statusCheckHandler( - options?: StatusCheckHandlerOptions, -): Promise; +export function statusCheckHandler(options?: StatusCheckHandlerOptions): Promise; // @public (undocumented) export interface StatusCheckHandlerOptions { - statusCheck?: StatusCheck; + statusCheck?: StatusCheck; } // @public export type UrlReader = { - read(url: string): Promise; - readTree(url: string, options?: ReadTreeOptions): Promise; - search(url: string, options?: SearchOptions): Promise; + read(url: string): Promise; + readTree(url: string, options?: ReadTreeOptions): Promise; + search(url: string, options?: SearchOptions): Promise; }; // @public export class UrlReaders { - static create({ logger, config, factories }: CreateOptions): UrlReader; - static default({ logger, config, factories }: CreateOptions): UrlReader; + static create({ logger, config, factories }: CreateOptions): UrlReader; + static default({ logger, config, factories }: CreateOptions): UrlReader; } // @public -export function useHotCleanup( - _module: NodeModule, - cancelEffect: () => void, -): void; +export function useHotCleanup(_module: NodeModule, cancelEffect: () => void): void; // @public export function useHotMemoize(_module: NodeModule, valueFactory: () => T): T; + // (No @packageDocumentation comment for this package) + ``` diff --git a/packages/backend-common/src/database/connection.ts b/packages/backend-common/src/database/connection.ts index 4a4a8c6ba2..cd92c3fcee 100644 --- a/packages/backend-common/src/database/connection.ts +++ b/packages/backend-common/src/database/connection.ts @@ -72,10 +72,7 @@ export async function ensureDatabaseExists( ) { const client: DatabaseClient = dbConfig.getString('client'); - return ConnectorMapping[client]?.ensureDatabaseExists?.( - dbConfig, - ...databases, - ); + ConnectorMapping[client]?.ensureDatabaseExists?.(dbConfig, ...databases); } /** From 4e0fdcb968bc14e0039f3b452e32e913fd874dc7 Mon Sep 17 00:00:00 2001 From: Minn Soe Date: Tue, 18 May 2021 23:13:03 +0100 Subject: [PATCH 15/93] test: add coverage for existing manager behaviour Signed-off-by: Minn Soe --- .../src/database/SingleConnection.test.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/packages/backend-common/src/database/SingleConnection.test.ts b/packages/backend-common/src/database/SingleConnection.test.ts index 3466b0f79d..231523c790 100644 --- a/packages/backend-common/src/database/SingleConnection.test.ts +++ b/packages/backend-common/src/database/SingleConnection.test.ts @@ -15,7 +15,7 @@ */ import { ConfigReader } from '@backstage/config'; -import { createDatabaseClient } from './connection'; +import { createDatabaseClient, ensureDatabaseExists } from './connection'; import { SingleConnectionDatabaseManager } from './SingleConnection'; jest.mock('./connection'); @@ -85,5 +85,13 @@ describe('SingleConnectionDatabaseManager', () => { plugin2CallArgs[1].connection.database, ); }); + + it('ensure plugin database is created', async () => { + await manager.forPlugin('test').getClient(); + const mockCalls = mocked(ensureDatabaseExists).mock.calls.splice(-1); + const [_, database] = mockCalls[0]; + + expect(database).toEqual('backstage_plugin_test'); + }); }); }); From 2ff816060d822cf9163e395367e29cecf76c77a6 Mon Sep 17 00:00:00 2001 From: Minn Soe Date: Tue, 18 May 2021 23:43:14 +0100 Subject: [PATCH 16/93] fix: ensure underlying connector promise passed up Signed-off-by: Minn Soe --- packages/backend-common/src/database/connection.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/backend-common/src/database/connection.ts b/packages/backend-common/src/database/connection.ts index cd92c3fcee..c9e0b3db48 100644 --- a/packages/backend-common/src/database/connection.ts +++ b/packages/backend-common/src/database/connection.ts @@ -69,10 +69,13 @@ export const createDatabase = createDatabaseClient; export async function ensureDatabaseExists( dbConfig: Config, ...databases: Array -) { +): Promise { const client: DatabaseClient = dbConfig.getString('client'); - ConnectorMapping[client]?.ensureDatabaseExists?.(dbConfig, ...databases); + return ConnectorMapping[client]?.ensureDatabaseExists?.( + dbConfig, + ...databases, + ); } /** From a9c4f0531faf3f7f2606d3b2d1fb5dc2d0f3ce32 Mon Sep 17 00:00:00 2001 From: Minn Soe Date: Wed, 19 May 2021 00:10:35 +0100 Subject: [PATCH 17/93] fix: sqlite3 memory typo and drop leading asterisk doc Signed-off-by: Minn Soe --- .changeset/five-donkeys-brake.md | 2 +- .../backend-common/src/database/PluginConnection.test.ts | 8 ++++---- packages/backend-common/src/database/PluginConnection.ts | 8 ++++---- packages/backend-common/src/database/connection.ts | 8 ++++---- packages/backend-common/src/database/mysql.ts | 2 +- packages/backend-common/src/database/postgres.ts | 2 +- packages/backend-common/src/database/sqlite3.ts | 2 +- 7 files changed, 16 insertions(+), 16 deletions(-) diff --git a/.changeset/five-donkeys-brake.md b/.changeset/five-donkeys-brake.md index 15d2cf7381..1e732b2b9a 100644 --- a/.changeset/five-donkeys-brake.md +++ b/.changeset/five-donkeys-brake.md @@ -27,7 +27,7 @@ backend: database: 'database_name_overriden' scaffolder: client: 'sqlite3' - connection: ':inmemory:' + connection: ':memory:' ``` Existing backstage installations can be migrated by swapping out the database diff --git a/packages/backend-common/src/database/PluginConnection.test.ts b/packages/backend-common/src/database/PluginConnection.test.ts index e3a5157d7d..51684210f8 100644 --- a/packages/backend-common/src/database/PluginConnection.test.ts +++ b/packages/backend-common/src/database/PluginConnection.test.ts @@ -83,7 +83,7 @@ describe('PluginConnectionDatabaseManager', () => { }, differentclientconnstring: { client: 'sqlite3', - connection: ':inmemory:', + connection: ':memory:', }, stringoverride: { connection: 'postgresql://testuser:testpass@acme:5432/userdbname', @@ -180,7 +180,7 @@ describe('PluginConnectionDatabaseManager', () => { backend: { database: { client: 'sqlite3', - connection: ':inmemory:', + connection: ':memory:', }, }, }), @@ -192,7 +192,7 @@ describe('PluginConnectionDatabaseManager', () => { expect(overrides).toHaveProperty( 'connection.filename', - expect.stringContaining(':inmemory:'), + expect.stringContaining(':memory:'), ); }); @@ -276,7 +276,7 @@ describe('PluginConnectionDatabaseManager', () => { expect(baseConfig.get().client).toEqual('sqlite3'); - expect(overrides).toHaveProperty('connection.filename', ':inmemory:'); + expect(overrides).toHaveProperty('connection.filename', ':memory:'); }); it('generates a database name override when prefix is not explicitly set', async () => { diff --git a/packages/backend-common/src/database/PluginConnection.ts b/packages/backend-common/src/database/PluginConnection.ts index 18c209c8d5..6db4e126b0 100644 --- a/packages/backend-common/src/database/PluginConnection.ts +++ b/packages/backend-common/src/database/PluginConnection.ts @@ -74,7 +74,7 @@ export class PluginConnectionDatabaseManager { * the value from `PluginConnectionDatabaseManager.DEFAULT_PREFIX`. * * @param pluginId Lookup the database name for given plugin - * */ + */ getDatabaseName(pluginId: string): string { const pluginConfig: Config = this.getConfigForPlugin(pluginId); @@ -83,7 +83,7 @@ export class PluginConnectionDatabaseManager { const rootSqliteName = typeof rootConnection === 'string' ? rootConnection - : this.config.getOptionalString('connection.filename') ?? ':inmemory:'; + : this.config.getOptionalString('connection.filename') ?? ':memory:'; const prefix = this.config.getOptionalString('prefix') ?? @@ -95,7 +95,7 @@ export class PluginConnectionDatabaseManager { pluginConfig.getOptionalString('connection.database') ?? // attempt to lookup sqlite3 database file name pluginConfig.getOptionalString('connection.filename') ?? - // if root is sqlite - attempt to use top level connection, fallback to :inmemory: + // if root is sqlite - attempt to use top level connection, fallback to :memory: (isSqlite ? rootSqliteName : null) ?? // generate a database name using prefix and pluginId `${prefix}${pluginId}` @@ -113,7 +113,7 @@ export class PluginConnectionDatabaseManager { * connection config will be extended with plugin specific config. * * @param pluginId The plugin that the database baseConfig should correspond to - * */ + */ private getConfigForPlugin(pluginId: string): Config { const pluginConfig = this.config.getOptionalConfig(pluginPath(pluginId)); diff --git a/packages/backend-common/src/database/connection.ts b/packages/backend-common/src/database/connection.ts index c9e0b3db48..005ed8819f 100644 --- a/packages/backend-common/src/database/connection.ts +++ b/packages/backend-common/src/database/connection.ts @@ -31,7 +31,7 @@ type DatabaseClient = 'pg' | 'sqlite3' | 'mysql' | 'mysql2' | string; * * Database connectors can be aliased here, for example mysql2 uses * the same connector as mysql. - * */ + */ const ConnectorMapping: Record = { pg: pgConnector, sqlite3: sqlite3Connector, @@ -80,7 +80,7 @@ export async function ensureDatabaseExists( /** * Provides a Knex.Config object with the provided database name for a given client. - * */ + */ export function createNameOverride( client: string, name: string, @@ -97,7 +97,7 @@ export function createNameOverride( /** * Parses a connection string for a given client and provides a connection config. - * */ + */ export function parseConnectionString( connectionString: string, client?: string, @@ -119,7 +119,7 @@ export function parseConnectionString( /** * Normalizes a connection config or string into an object which can be passed to Knex. - * */ + */ export function normalizeConnection( connection: Knex.StaticConnectionConfig | JsonObject | string, client: string, diff --git a/packages/backend-common/src/database/mysql.ts b/packages/backend-common/src/database/mysql.ts index 3bd874de11..beff5d9f82 100644 --- a/packages/backend-common/src/database/mysql.ts +++ b/packages/backend-common/src/database/mysql.ts @@ -173,7 +173,7 @@ export function createMysqlNameOverride(name: string): Partial { * MySql database connector. * * Exposes database connector functionality via an immutable object. - * */ + */ export const mysqlConnector: DatabaseConnector = Object.freeze({ createClient: createMysqlDatabaseClient, ensureDatabaseExists: ensureMysqlDatabaseExists, diff --git a/packages/backend-common/src/database/postgres.ts b/packages/backend-common/src/database/postgres.ts index 40c6a41504..000b84460c 100644 --- a/packages/backend-common/src/database/postgres.ts +++ b/packages/backend-common/src/database/postgres.ts @@ -145,7 +145,7 @@ export function createPgNameOverride(name: string): Partial { * PostgreSQL database connector. * * Exposes database connector functionality via an immutable object. - * */ + */ export const pgConnector: DatabaseConnector = Object.freeze({ createClient: createPgDatabaseClient, ensureDatabaseExists: ensurePgDatabaseExists, diff --git a/packages/backend-common/src/database/sqlite3.ts b/packages/backend-common/src/database/sqlite3.ts index 98c39b9c97..83f5f60699 100644 --- a/packages/backend-common/src/database/sqlite3.ts +++ b/packages/backend-common/src/database/sqlite3.ts @@ -119,7 +119,7 @@ export function parseSqliteConnectionString( * Sqlite3 database connector. * * Exposes database connector functionality via an immutable object. - * */ + */ export const sqlite3Connector: DatabaseConnector = Object.freeze({ createClient: createSqliteDatabaseClient, createNameOverride: createSqliteNameOverride, From ef1d705b76d0732316fda91789f8c48b98c2c718 Mon Sep 17 00:00:00 2001 From: Minn Soe Date: Wed, 19 May 2021 00:22:44 +0100 Subject: [PATCH 18/93] fix: make plugin database getDatabaseName private Signed-off-by: Minn Soe --- packages/backend-common/api-report.md | 1 - packages/backend-common/src/database/PluginConnection.ts | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/backend-common/api-report.md b/packages/backend-common/api-report.md index 002cfaac89..b5e4b0a84e 100644 --- a/packages/backend-common/api-report.md +++ b/packages/backend-common/api-report.md @@ -273,7 +273,6 @@ export class PluginConnectionDatabaseManager { static readonly DEFAULT_PREFIX = "backstage_plugin_"; forPlugin(pluginId: string): PluginDatabaseManager; static fromConfig(config: Config): PluginConnectionDatabaseManager; - getDatabaseName(pluginId: string): string; } // @public diff --git a/packages/backend-common/src/database/PluginConnection.ts b/packages/backend-common/src/database/PluginConnection.ts index 6db4e126b0..4f41bd068e 100644 --- a/packages/backend-common/src/database/PluginConnection.ts +++ b/packages/backend-common/src/database/PluginConnection.ts @@ -75,7 +75,7 @@ export class PluginConnectionDatabaseManager { * * @param pluginId Lookup the database name for given plugin */ - getDatabaseName(pluginId: string): string { + private getDatabaseName(pluginId: string): string { const pluginConfig: Config = this.getConfigForPlugin(pluginId); // determine root sqlite config to pass through as this is a special case From 2976f3bae349bb37de3f20d8ae2a1cc8456f9ead Mon Sep 17 00:00:00 2001 From: Minn Soe Date: Tue, 25 May 2021 18:05:21 +0100 Subject: [PATCH 19/93] refactor: rename and deprecate database manager Changes: - Deprecates SingleConnectionManager and aliases to DatabaseManager. - Simplifies database config typing in `config.d.ts`. - Drops implementation specific assert in SingleConnectionManager test. - Move database prefix reference and drop static property for default value. Signed-off-by: Minn Soe --- .changeset/five-donkeys-brake.md | 15 ++--- packages/backend-common/api-report.md | 21 +++--- packages/backend-common/config.d.ts | 67 ++++++++----------- ...ection.test.ts => DatabaseManager.test.ts} | 57 +++++++--------- ...PluginConnection.ts => DatabaseManager.ts} | 30 ++++----- .../src/database/SingleConnection.test.ts | 10 +-- .../src/database/SingleConnection.ts | 63 ++--------------- packages/backend-common/src/database/index.ts | 2 +- packages/backend/src/index.ts | 4 +- 9 files changed, 97 insertions(+), 172 deletions(-) rename packages/backend-common/src/database/{PluginConnection.test.ts => DatabaseManager.test.ts} (88%) rename packages/backend-common/src/database/{PluginConnection.ts => DatabaseManager.ts} (88%) diff --git a/.changeset/five-donkeys-brake.md b/.changeset/five-donkeys-brake.md index 1e732b2b9a..d5bc695714 100644 --- a/.changeset/five-donkeys-brake.md +++ b/.changeset/five-donkeys-brake.md @@ -1,11 +1,10 @@ --- -'example-backend': minor '@backstage/backend-common': minor --- -Introduces `PluginConnectionDatabaseManager`, a backwards compatible database -connection manager which allows developers to configure database connections on -a per plugin basis. +Deprecates `SingleConnectionDatabaseManager` and provides an API compatible database +connection manager, `DatabaseManager`, which allows developers to configure database +connections on a per plugin basis. The `backend.database` config path allows you to set `prefix` to use an alternate prefix for automatically generated database names, the default is @@ -30,13 +29,13 @@ backend: connection: ':memory:' ``` -Existing backstage installations can be migrated by swapping out the database -manager under `packages/backend/src/index.ts` as shown below: +Migrate existing backstage installations by swapping out the database manager in the +`packages/backend/src/index.ts` file as shown below: ```diff import { - SingleConnectionDatabaseManager, -+ PluginConnectionDatabaseManager, ++ DatabaseManager, } from '@backstage/backend-common'; // ... @@ -44,7 +43,7 @@ import { function makeCreateEnv(config: Config) { // ... - const databaseManager = SingleConnectionDatabaseManager.fromConfig(config); -+ const databaseManager = PluginConnectionDatabaseManager.fromConfig(config); ++ const databaseManager = DatabaseManager.fromConfig(config); // ... } ``` diff --git a/packages/backend-common/api-report.md b/packages/backend-common/api-report.md index b5e4b0a84e..d4d075004a 100644 --- a/packages/backend-common/api-report.md +++ b/packages/backend-common/api-report.md @@ -104,6 +104,12 @@ export function createServiceBuilder(_module: NodeModule): ServiceBuilderImpl; // @public (undocumented) export function createStatusCheckRouter(options: StatusCheckRouterOptions): Promise; +// @public (undocumented) +export class DatabaseManager { + forPlugin(pluginId: string): PluginDatabaseManager; + static fromConfig(config: Config): DatabaseManager; + } + // @public (undocumented) export class DockerContainerRunner implements ContainerRunner { constructor({ dockerClient }: { @@ -267,14 +273,6 @@ export type PluginCacheManager = { getClient: (options?: ClientOptions) => CacheClient; }; -// @public (undocumented) -export class PluginConnectionDatabaseManager { - // (undocumented) - static readonly DEFAULT_PREFIX = "backstage_plugin_"; - forPlugin(pluginId: string): PluginDatabaseManager; - static fromConfig(config: Config): PluginConnectionDatabaseManager; - } - // @public export interface PluginDatabaseManager { getClient(): Promise; @@ -344,11 +342,8 @@ export type ServiceBuilder = { // @public (undocumented) export function setRootLogger(newLogger: winston.Logger): void; -// @public -export class SingleConnectionDatabaseManager { - forPlugin(pluginId: string): PluginDatabaseManager; - static fromConfig(config: Config): SingleConnectionDatabaseManager; - } +// @public @deprecated +export const SingleConnectionDatabaseManager: typeof DatabaseManager; // @public export class SingleHostDiscovery implements PluginEndpointDiscovery { diff --git a/packages/backend-common/config.d.ts b/packages/backend-common/config.d.ts index d735f39935..3ada25dd8a 100644 --- a/packages/backend-common/config.d.ts +++ b/packages/backend-common/config.d.ts @@ -14,22 +14,15 @@ * limitations under the License. */ -export type PluginDatabaseConfig = - | { - /** Database client to use for plugin. */ - client?: 'sqlite3'; - /** Database connection to use with plugin. */ - connection?: ':memory:' | string | { filename: string }; - } - | { - /** Database client to use for plugin. */ - client?: 'pg'; - /** - * PostgreSQL connection string or knex configuration object for plugin. - * @secret - */ - connection?: string | object; - }; +export type PluginDatabaseConfig = { + /** Database client to use. */ + client?: 'sqlite3' | 'pg'; + /** + * Database connection to use. + * @secret + */ + connection?: string | object; +}; export interface Config { app: { @@ -70,32 +63,30 @@ export interface Config { }; }; - /** Database connection configuration, select database type using the `client` field */ - database: - | { - client: 'sqlite3'; - connection: ':memory:' | string | { filename: string }; - /** Optional sqlite3 database filename prefix. */ - prefix?: string; - /** Override database config per plugin. */ - plugin?: { - [pluginId: string]: PluginDatabaseConfig; - }; - } - | { - client: 'pg'; + /** Database connection configuration, select base database type using the `client` field */ + database: { + /** Default database client to use */ + client: 'sqlite3' | 'pg'; + /** + * Base database connection string or Knex object + * @secret + */ + connection: string | object; + /** Database name prefix override */ + prefix?: string; + /** Plugin specific database configuration and client override */ + plugin?: { + [pluginId: string]: { + /** Database client override */ + client?: 'sqlite3' | 'pg'; /** - * PostgreSQL connection string or knex configuration object. + * Database connection string or Knex object override * @secret */ - connection: string | object; - /** Optional PostgreSQL database prefix. */ - prefix?: string; - /** Override database config per plugin. */ - plugin?: { - [pluginId: string]: PluginDatabaseConfig; - }; + connection?: string | object; }; + }; + }; /** Cache connection configuration, select cache type using the `store` field */ cache?: diff --git a/packages/backend-common/src/database/PluginConnection.test.ts b/packages/backend-common/src/database/DatabaseManager.test.ts similarity index 88% rename from packages/backend-common/src/database/PluginConnection.test.ts rename to packages/backend-common/src/database/DatabaseManager.test.ts index 51684210f8..8f3331403d 100644 --- a/packages/backend-common/src/database/PluginConnection.test.ts +++ b/packages/backend-common/src/database/DatabaseManager.test.ts @@ -16,7 +16,7 @@ import { ConfigReader } from '@backstage/config'; import { omit } from 'lodash'; import { createDatabaseClient, ensureDatabaseExists } from './connection'; -import { PluginConnectionDatabaseManager } from './PluginConnection'; +import { DatabaseManager } from './DatabaseManager'; jest.mock('./connection', () => ({ ...jest.requireActual('./connection'), @@ -24,40 +24,35 @@ jest.mock('./connection', () => ({ ensureDatabaseExists: jest.fn(), })); -describe('PluginConnectionDatabaseManager', () => { +describe('DatabaseManager', () => { // This is similar to the ts-jest `mocked` helper. const mocked = (f: Function) => f as jest.Mock; afterEach(() => jest.resetAllMocks()); - describe('PluginConnectionDatabaseManager.fromConfig', () => { - const backendConfig = { - backend: { - database: { - client: 'pg', - connection: { - host: 'localhost', - user: 'foo', - password: 'bar', - database: 'foodb', + describe('DatabaseManager.fromConfig', () => { + it('accesses the backend.database key', () => { + const config = new ConfigReader({ + backend: { + database: { + client: 'pg', + connection: { + host: 'localhost', + user: 'foo', + password: 'bar', + database: 'foodb', + }, }, }, - }, - }; - const defaultConfig = () => new ConfigReader(backendConfig); + }); + const getConfigSpy = jest.spyOn(config, 'getConfig'); + DatabaseManager.fromConfig(config); - it('accesses the backend.database key', () => { - const getConfig = jest.fn(); - const config = defaultConfig(); - config.getConfig = getConfig; - - PluginConnectionDatabaseManager.fromConfig(config); - - expect(getConfig.mock.calls[0][0]).toEqual('backend.database'); + expect(getConfigSpy).toHaveBeenCalledWith('backend.database'); }); }); - describe('PluginConnectionDatabaseManager.forPlugin', () => { + describe('DatabaseManager.forPlugin', () => { const config = { backend: { database: { @@ -92,9 +87,7 @@ describe('PluginConnectionDatabaseManager', () => { }, }, }; - const manager = PluginConnectionDatabaseManager.fromConfig( - new ConfigReader(config), - ); + const manager = DatabaseManager.fromConfig(new ConfigReader(config)); it('connects to a plugin database using default config', async () => { const pluginId = 'pluginwithoutconfig'; @@ -120,7 +113,7 @@ describe('PluginConnectionDatabaseManager', () => { }); it('provides a plugin db which uses components from top level connection string', async () => { - const testManager = PluginConnectionDatabaseManager.fromConfig( + const testManager = DatabaseManager.fromConfig( new ConfigReader({ backend: { database: { @@ -153,7 +146,7 @@ describe('PluginConnectionDatabaseManager', () => { }); it('uses top level sqlite database filename if plugin config is not present', async () => { - const testManager = PluginConnectionDatabaseManager.fromConfig( + const testManager = DatabaseManager.fromConfig( new ConfigReader({ backend: { database: { @@ -175,7 +168,7 @@ describe('PluginConnectionDatabaseManager', () => { }); it('provides an inmemory sqlite database if top level is also inmemory and plugin config is not present', async () => { - const testManager = PluginConnectionDatabaseManager.fromConfig( + const testManager = DatabaseManager.fromConfig( new ConfigReader({ backend: { database: { @@ -280,7 +273,7 @@ describe('PluginConnectionDatabaseManager', () => { }); it('generates a database name override when prefix is not explicitly set', async () => { - const testManager = PluginConnectionDatabaseManager.fromConfig( + const testManager = DatabaseManager.fromConfig( new ConfigReader({ backend: { database: { @@ -302,7 +295,7 @@ describe('PluginConnectionDatabaseManager', () => { expect(overrides).toHaveProperty( 'connection.database', - expect.stringContaining(PluginConnectionDatabaseManager.DEFAULT_PREFIX), + expect.stringContaining('backstage_plugin_'), ); }); diff --git a/packages/backend-common/src/database/PluginConnection.ts b/packages/backend-common/src/database/DatabaseManager.ts similarity index 88% rename from packages/backend-common/src/database/PluginConnection.ts rename to packages/backend-common/src/database/DatabaseManager.ts index 4f41bd068e..34ceea55e3 100644 --- a/packages/backend-common/src/database/PluginConnection.ts +++ b/packages/backend-common/src/database/DatabaseManager.ts @@ -28,11 +28,9 @@ function pluginPath(pluginId: string): string { return `plugin.${pluginId}`; } -export class PluginConnectionDatabaseManager { - static readonly DEFAULT_PREFIX = 'backstage_plugin_'; - +export class DatabaseManager { /** - * Creates a PluginConnectionDatabaseManager from `backend.database` config. + * Creates a DatabaseManager from `backend.database` config. * * The database manager allows the user to set connection and client settings on a per pluginId * basis by defining a database config block under `plugin.` in addition to top level @@ -41,13 +39,19 @@ export class PluginConnectionDatabaseManager { * * @param config The loaded application configuration. */ - static fromConfig(config: Config): PluginConnectionDatabaseManager { - return new PluginConnectionDatabaseManager( - config.getConfig('backend.database'), + static fromConfig(config: Config): DatabaseManager { + const databaseConfig = config.getConfig('backend.database'); + + return new DatabaseManager( + databaseConfig, + databaseConfig.getOptionalString('prefix'), ); } - private constructor(private readonly config: Config) {} + private constructor( + private readonly config: Config, + private readonly prefix: string = 'backstage_plugin_', + ) {} /** * Generates a PluginDatabaseManager for consumption by plugins. @@ -70,8 +74,8 @@ export class PluginConnectionDatabaseManager { * * This method provides the effective database name which is determined using global * and plugin specific database config. If no explicit database name is configured, - * this method will provide a generated name which is the pluginId prefixed using - * the value from `PluginConnectionDatabaseManager.DEFAULT_PREFIX`. + * this method will provide a generated name which is the pluginId prefixed with + * 'backstage_plugin_'. * * @param pluginId Lookup the database name for given plugin */ @@ -85,10 +89,6 @@ export class PluginConnectionDatabaseManager { ? rootConnection : this.config.getOptionalString('connection.filename') ?? ':memory:'; - const prefix = - this.config.getOptionalString('prefix') ?? - PluginConnectionDatabaseManager.DEFAULT_PREFIX; - const isSqlite = this.config.getString('client') === 'sqlite3'; return ( // attempt to lookup pg and mysql database name @@ -98,7 +98,7 @@ export class PluginConnectionDatabaseManager { // if root is sqlite - attempt to use top level connection, fallback to :memory: (isSqlite ? rootSqliteName : null) ?? // generate a database name using prefix and pluginId - `${prefix}${pluginId}` + `${this.prefix}${pluginId}` ); } diff --git a/packages/backend-common/src/database/SingleConnection.test.ts b/packages/backend-common/src/database/SingleConnection.test.ts index 231523c790..852dd33944 100644 --- a/packages/backend-common/src/database/SingleConnection.test.ts +++ b/packages/backend-common/src/database/SingleConnection.test.ts @@ -18,7 +18,11 @@ import { ConfigReader } from '@backstage/config'; import { createDatabaseClient, ensureDatabaseExists } from './connection'; import { SingleConnectionDatabaseManager } from './SingleConnection'; -jest.mock('./connection'); +jest.mock('./connection', () => ({ + ...jest.requireActual('./connection'), + createDatabaseClient: jest.fn(), + ensureDatabaseExists: jest.fn(), +})); describe('SingleConnectionDatabaseManager', () => { const defaultConfigOptions = { @@ -43,9 +47,8 @@ describe('SingleConnectionDatabaseManager', () => { describe('SingleConnectionDatabaseManager.fromConfig', () => { it('accesses the backend.database key', () => { - const getConfig = jest.fn(); const config = defaultConfig(); - config.getConfig = getConfig; + const getConfig = jest.spyOn(config, 'getConfig'); SingleConnectionDatabaseManager.fromConfig(config); @@ -64,7 +67,6 @@ describe('SingleConnectionDatabaseManager', () => { const mockCalls = mocked(createDatabaseClient).mock.calls.splice(-1); const callArgs = mockCalls[0]; - expect(callArgs[0].get()).toEqual(defaultConfigOptions.backend.database); expect(callArgs[1].connection.database).toEqual( `backstage_plugin_${pluginId}`, ); diff --git a/packages/backend-common/src/database/SingleConnection.ts b/packages/backend-common/src/database/SingleConnection.ts index 1c5931a662..5bdd99c0ae 100644 --- a/packages/backend-common/src/database/SingleConnection.ts +++ b/packages/backend-common/src/database/SingleConnection.ts @@ -14,69 +14,14 @@ * limitations under the License. */ -import { Knex } from 'knex'; -import { Config } from '@backstage/config'; -import { createDatabaseClient, ensureDatabaseExists } from './connection'; -import { PluginDatabaseManager } from './types'; +import { DatabaseManager } from './DatabaseManager'; /** * Implements a Database Manager which will automatically create new databases * for plugins when requested. All requested databases are created with the * credentials provided; if the database already exists no attempt to create * the database will be made. + * + * @deprecated Use `DatabaseManager` from `@backend-common` instead. */ -export class SingleConnectionDatabaseManager { - /** - * Creates a new SingleConnectionDatabaseManager instance by reading from the `backend` - * config section, specifically the `.database` key for discovering the management - * database configuration. - * - * @param config The loaded application configuration. - */ - static fromConfig(config: Config): SingleConnectionDatabaseManager { - return new SingleConnectionDatabaseManager( - config.getConfig('backend.database'), - ); - } - - private constructor(private readonly config: Config) {} - - /** - * Generates a PluginDatabaseManager for consumption by plugins. - * - * @param pluginId The plugin that the database manager should be created for. Plugin names should be unique. - */ - forPlugin(pluginId: string): PluginDatabaseManager { - const _this = this; - - return { - getClient(): Promise { - return _this.getDatabase(pluginId); - }, - }; - } - - private async getDatabase(pluginId: string): Promise { - const config = this.config; - const overrides = SingleConnectionDatabaseManager.getDatabaseOverrides( - pluginId, - ); - const overrideConfig = overrides.connection as Knex.ConnectionConfig; - await this.ensureDatabase(overrideConfig.database); - - return createDatabaseClient(config, overrides); - } - - private static getDatabaseOverrides(pluginId: string): Knex.Config { - return { - connection: { - database: `backstage_plugin_${pluginId}`, - }, - }; - } - - private async ensureDatabase(database: string) { - const config = this.config; - await ensureDatabaseExists(config, database); - } -} +export const SingleConnectionDatabaseManager = DatabaseManager; diff --git a/packages/backend-common/src/database/index.ts b/packages/backend-common/src/database/index.ts index 7a8a81e187..7dfb08359b 100644 --- a/packages/backend-common/src/database/index.ts +++ b/packages/backend-common/src/database/index.ts @@ -17,4 +17,4 @@ export * from './connection'; export * from './types'; export * from './SingleConnection'; -export * from './PluginConnection'; +export * from './DatabaseManager'; diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index 9cab64e797..67149c8163 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -29,7 +29,7 @@ import { getRootLogger, loadBackendConfig, notFoundHandler, - PluginConnectionDatabaseManager, + DatabaseManager, SingleHostDiscovery, UrlReaders, useHotMemoize, @@ -59,7 +59,7 @@ function makeCreateEnv(config: Config) { root.info(`Created UrlReader ${reader}`); - const databaseManager = PluginConnectionDatabaseManager.fromConfig(config); + const databaseManager = DatabaseManager.fromConfig(config); const cacheManager = CacheManager.fromConfig(config); return (plugin: string): PluginEnvironment => { From 5a3ce340728f4bc51aa58fc688db71f38f5e155e Mon Sep 17 00:00:00 2001 From: Minn Soe Date: Tue, 25 May 2021 18:28:05 +0100 Subject: [PATCH 20/93] refactor: migrate from deprecated database manager Changes: - Swaps out `SingleConnectionDatabaseManager` to `DatabaseManager` across the repo. - Updates `backend-test-utils` to generate test plugin names prefixed with db to satisfy plugin naming constraint, e.g. 0 becomes db0. Signed-off-by: Minn Soe --- .changeset/five-donkeys-brake.md | 1 + .../tutorials/configuring-plugin-databases.md | 222 +++++++++++------- packages/backend-common/config.d.ts | 10 - .../src/database/TestDatabases.test.ts | 6 +- .../src/database/TestDatabases.ts | 12 +- .../backend-test-utils/src/database/types.ts | 5 +- .../default-app/packages/backend/src/index.ts | 4 +- .../src/service/CodeCoverageDatabase.test.ts | 4 +- .../src/service/router.test.ts | 4 +- .../tasks/StorageTaskBroker.test.ts | 7 +- .../src/scaffolder/tasks/TaskWorker.test.ts | 9 +- .../src/service/router.test.ts | 4 +- 12 files changed, 168 insertions(+), 120 deletions(-) diff --git a/.changeset/five-donkeys-brake.md b/.changeset/five-donkeys-brake.md index d5bc695714..8a6523bb61 100644 --- a/.changeset/five-donkeys-brake.md +++ b/.changeset/five-donkeys-brake.md @@ -1,5 +1,6 @@ --- '@backstage/backend-common': minor +'@backstage/create-app': minor --- Deprecates `SingleConnectionDatabaseManager` and provides an API compatible database diff --git a/docs/tutorials/configuring-plugin-databases.md b/docs/tutorials/configuring-plugin-databases.md index b65addd7a8..27b800b3d2 100644 --- a/docs/tutorials/configuring-plugin-databases.md +++ b/docs/tutorials/configuring-plugin-databases.md @@ -1,42 +1,40 @@ --- id: configuring-plugin-databases -title: Configuring Plugin Specific Databases +title: Configuring Plugin Databases # prettier-ignore -description: Guide on how to use predefined databases for each plugin. +description: Guide on how to configure Backstage databases. --- -There are occasions where it may be difficult to deploy Backstage with -automatically created databases in production due to access control or other -restrictions. For example, your infrastructure might be defined as code using -tools such as Terraform or AWS CloudFormation where the name of each database is -defined, created and assigned explicitly. You may also need to use different -credentials for each database or use a set of credentials without the -permissions needed to create databases. +This guide covers a variety of production persistence use cases which are +supported out of the box by Backstage. The database manager allows the developer +to set the client and database connection details on a per plugin basis in +addition to the base client and connection configuration. This means that you +can use a SQLite 3 in-memory database for a specific plugin whilst using +PostgreSQL for everything else and so on. -`@backstage/backend-common` provides an alternate database manager, -`PluginConnectionDatabaseManager`, which allows the developer to set the client -and database connection on a per plugin basis in addition to the default client -and connection configuration. This means that you can use a `sqlite3` in memory -database for a specific plugin whilst using `postgres` for everything else and -so on. +By default, Backstage uses automatically created databases for each plugin whose +names follow the `backstage_plugin_` pattern, e.g. +`backstage_plugin_auth`. You can configure a different database name prefix for +use cases where you have multiple deployments running on a shared database +instance or cluster. -The database manager also allows you to change the database name prefix which is -used when a plugin database isn't explicitly configured. +With infrastructure defined as code or data (Terraform, AWS CloudFormation, +etc.), you may have database credentials which lack permissions to create new +databases or you do not have control over the database names. In these +instances, you can set the database name and connection information on a per +plugin basis as mentioned earlier. -There are two additional configuration options for this database manager: +Backstage supports all of these use cases with the `DatabaseManager` provided by +`@backstage/backend-common`. We will now cover how to use and configure +Backstage's databases. -- **`backend.database.prefix`:** is used to override the default - `backstage_plugin_` prefix which is used to generate a database name when it - is not explicitly set for that plugin. -- **`backend.database.plugin.`:** is used to define a `client` and - `connection` block for the plugin matching the `pluginId`, e.g. `catalog` is - the `pluginId` for the catalog plugin and any configuration defined under that - block is specific to that plugin. +## Prerequisites -## Install Database Drivers +### Dependencies -If you intend to use both `postgres` and `sqlite3`, you need to make sure the -appropriate database drivers are installed in your `backend` package. +Please ensure the appropriate database drivers are installed in your `backend` +package. If you intend to use both `postgres` and `sqlite3`, you can install +both of them. ```shell cd packages/backend @@ -51,48 +49,17 @@ yarn add sqlite3 From an operational perspective, you only need to install drivers for clients that are actively used. -## Add Configuration +### Database Manager -You can set the same type of values for `backend.database..client` and -`backend.database..connection` which are also accepted at the top -level. - -It is possible to override the default database name prefix, -`backstage_plugin_`, which is used when a name isn't explicitly defined. Set -`backend.database.prefix` as shown below. The database names for plugins such as -`catalog` and `auth` would now be `my_company_catalog` and `my_company_auth` -instead of `backstage_plugin_catalog` and `backstage_plugin_auth`. - -```yaml -backend: - database: - client: pg - prefix: my_company_ - connection: - host: localhost - user: postgres - password: password - plugin: - code-coverage: - connection: - database: pg_code_coverage_set_by_user -``` - -In the example above, the `code-coverage` plugin will use the same connection -configuration defined under `database.connection` and use -`pg_code_coverage_set_by_user` instead of `my_company_code-coverage` which would -be automatically generated if a plugin configuration wasn't explicitly set. - -## Integrate `PluginConnectionDatabaseManager` into `backend` - -The `SingleConnectionDatabaseManager` used by default should be replaced with -the `PluginConnectionDatabaseManager` in your `packages/backend/src/index.ts` -file. Import the manager and replace the `.fromConfig` call as shown below: +Existing Backstage instances should be updated to use `DatabaseManager` from +`@backstage/backend-common` in your `packages/backend/src/index.ts` file, the +`SingleConnectionDatabaseManager` has been deprecated. Import the manager and +update the references as shown below if this is not the case: ```diff import { - SingleConnectionDatabaseManager, -+ PluginConnectionDatabaseManager, ++ DatabaseManager, } from '@backstage/backend-common'; // ... @@ -100,24 +67,121 @@ import { function makeCreateEnv(config: Config) { // ... - const databaseManager = SingleConnectionDatabaseManager.fromConfig(config); -+ const databaseManager = PluginConnectionDatabaseManager.fromConfig(config); ++ const databaseManager = DatabaseManager.fromConfig(config); // ... } ``` +## Configuration + +You should set the base database client and connection information in your +`app-config.yaml` (or equivalent) file. The base client and configuration is +used as the default which is extended for each plugin with the same or unset +client type. If a client type is specified for a specific plugin which does not +match the base client, the configuration set for the plugin will be used as is +without extending the base configuration. + +Client type and configuration for plugins need to be defined under +**`backend.database.plugin.`**. As an example, `catalog` is the +`pluginId` for the catalog plugin and any configuration defined under that block +is specific to that plugin. We will now explore more detailed example +configurations below. + +### Minimal In-Memory Configuration + +In the example below, we are using `sqlite3` in-memory databases for all +plugins. You may want to use this configuration for testing or other non-durable +use cases. + +```yaml +backend: + database: + client: sqlite3 + connection: ':memory:' +``` + +### PostgreSQL + +The example below uses PostgreSQL (`pg`) as the database client for all plugins. +The `auth` plugin uses a user defined database name instead of the automatically +generated one which would have been `backstage_plugin_auth`. + +```yaml +backend: + database: + client: pg + connection: + host: some.example-pg-instance.tld + user: postgres + password: password + port: 5432 + plugin: + auth: + connection: + database: pg_auth_set_by_user +``` + +### Custom Database Name Prefix + +The configuration below uses `example_prefix_` as the database name prefix +instead of `backstage_plugin_`. Plugins such as `auth` and `catalog` will use +databases named `example_prefix_auth` and `example_prefix_catalog` respectively. + +```yaml +backend: + database: + client: pg + connection: + host: some.example-pg-instance.tld + user: postgres + password: password + port: 5432 + prefix: 'example_prefix_' +``` + +### Connection Configuration Per Plugin + +Both `auth` and `catalog` use connection configuration with different +credentials and database names. This type of configuration can be useful for +environments with infrastructure as code or data which may provide randomly +generated credentials and/or database names. + +```yaml +backend: + database: + client: pg + connection: 'postgresql://some.example-pg-instance.tld:5432' + plugin: + auth: + connection: 'postgresql://fort:knox@some.example-pg-instance.tld:5432/unwitting_fox_jumps' + catalog: + connection: 'postgresql://bank:reserve@some.example-pg-instance.tld:5432/shuffle_ransack_playback' +``` + +### PostgreSQL and SQLite 3 + +The example below uses PostgreSQL (`pg`) as the database client for all plugins +except the `auth` plugin which uses `sqlite3`. As the `auth` plugin's client +type is different from the base client type, the connection configuration for +`auth` is used verbatim without extending the base configuration for PostgreSQL. + +```yaml +backend: + database: + client: pg + connection: 'postgresql://foo:bar@some.example-pg-instance.tld:5432' + plugin: + auth: + client: sqlite3 + connection: ':memory:' +``` + ## Check Your Databases -The `PluginConnectionDatabaseManager` preserves the behaviour of the -`SingleConnectionDatabaseManager`. If the database does not exist, it will -attempt to create it. +The `DatabaseManager` will attempt to create the databases if they do not exist. +If you have set credentials per plugin because the credentials in the base +configuration do not have permissions to create databases, you must ensure they +exist before starting the service. The service will not be able to create them, +it can only use them. -If you are using this database manager to set the database name upfront because -the credentials do not have permissions to create databases, you must ensure -they exist before starting the service. The service will not be able to create -them, it can only use them. - -`sqlite3` databases do not need to be created upfront as with the existing -database manager. - -Your Backstage App can now use different database clients and configuration per -plugin! +Good luck! diff --git a/packages/backend-common/config.d.ts b/packages/backend-common/config.d.ts index 3ada25dd8a..b42ce3eca5 100644 --- a/packages/backend-common/config.d.ts +++ b/packages/backend-common/config.d.ts @@ -14,16 +14,6 @@ * limitations under the License. */ -export type PluginDatabaseConfig = { - /** Database client to use. */ - client?: 'sqlite3' | 'pg'; - /** - * Database connection to use. - * @secret - */ - connection?: string | object; -}; - export interface Config { app: { baseUrl: string; // defined in core, but repeated here without doc diff --git a/packages/backend-test-utils/src/database/TestDatabases.test.ts b/packages/backend-test-utils/src/database/TestDatabases.test.ts index 7c1e6e1bdd..7a51111b02 100644 --- a/packages/backend-test-utils/src/database/TestDatabases.test.ts +++ b/packages/backend-test-utils/src/database/TestDatabases.test.ts @@ -71,7 +71,7 @@ describe('TestDatabases', () => { await input.insert({ x: 'y' }).into('a'); // Look for the mark - const database = 'backstage_plugin_0'; + const database = 'backstage_plugin_db0'; const output = knexFactory({ client: 'pg', connection: { host, port, user, password, database }, @@ -105,7 +105,7 @@ describe('TestDatabases', () => { await input.insert({ x: 'y' }).into('a'); // Look for the mark - const database = 'backstage_plugin_0'; + const database = 'backstage_plugin_db0'; const output = knexFactory({ client: 'pg', connection: { host, port, user, password, database }, @@ -139,7 +139,7 @@ describe('TestDatabases', () => { await input.insert({ x: 'y' }).into('a'); // Look for the mark - const database = 'backstage_plugin_0'; + const database = 'backstage_plugin_db0'; const output = knexFactory({ client: 'mysql2', connection: { host, port, user, password, database }, diff --git a/packages/backend-test-utils/src/database/TestDatabases.ts b/packages/backend-test-utils/src/database/TestDatabases.ts index ab5846d84d..0ba6bffc0b 100644 --- a/packages/backend-test-utils/src/database/TestDatabases.ts +++ b/packages/backend-test-utils/src/database/TestDatabases.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { SingleConnectionDatabaseManager } from '@backstage/backend-common'; +import { DatabaseManager } from '@backstage/backend-common'; import { ConfigReader } from '@backstage/config'; import { Knex } from 'knex'; import { isDockerDisabledForTests } from '../util/isDockerDisabledForTests'; @@ -142,7 +142,7 @@ export class TestDatabases { // Ensure that a unique logical database is created in the instance const connection = await instance.databaseManager - .forPlugin(String(this.lastDatabaseIndex++)) + .forPlugin(String(`db${this.lastDatabaseIndex++}`)) .getClient(); instance.connections.push(connection); @@ -157,7 +157,7 @@ export class TestDatabases { if (envVarName) { const connectionString = process.env[envVarName]; if (connectionString) { - const databaseManager = SingleConnectionDatabaseManager.fromConfig( + const databaseManager = DatabaseManager.fromConfig( new ConfigReader({ backend: { database: { @@ -195,7 +195,7 @@ export class TestDatabases { properties.dockerImageName!, ); - const databaseManager = SingleConnectionDatabaseManager.fromConfig( + const databaseManager = DatabaseManager.fromConfig( new ConfigReader({ backend: { database: { @@ -220,7 +220,7 @@ export class TestDatabases { properties.dockerImageName!, ); - const databaseManager = SingleConnectionDatabaseManager.fromConfig( + const databaseManager = DatabaseManager.fromConfig( new ConfigReader({ backend: { database: { @@ -241,7 +241,7 @@ export class TestDatabases { private async initSqlite( _properties: TestDatabaseProperties, ): Promise { - const databaseManager = SingleConnectionDatabaseManager.fromConfig( + const databaseManager = DatabaseManager.fromConfig( new ConfigReader({ backend: { database: { diff --git a/packages/backend-test-utils/src/database/types.ts b/packages/backend-test-utils/src/database/types.ts index 791cf09b7b..91b5939765 100644 --- a/packages/backend-test-utils/src/database/types.ts +++ b/packages/backend-test-utils/src/database/types.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { SingleConnectionDatabaseManager } from '@backstage/backend-common'; +import { DatabaseManager } from '@backstage/backend-common'; import { Knex } from 'knex'; /** @@ -35,10 +35,9 @@ export type TestDatabaseProperties = { export type Instance = { stopContainer?: () => Promise; - databaseManager: SingleConnectionDatabaseManager; + databaseManager: DatabaseManager; connections: Array; }; - export const allDatabases: Record< TestDatabaseId, TestDatabaseProperties diff --git a/packages/create-app/templates/default-app/packages/backend/src/index.ts b/packages/create-app/templates/default-app/packages/backend/src/index.ts index aebd034aae..70f4fb676e 100644 --- a/packages/create-app/templates/default-app/packages/backend/src/index.ts +++ b/packages/create-app/templates/default-app/packages/backend/src/index.ts @@ -14,7 +14,7 @@ import { useHotMemoize, notFoundHandler, CacheManager, - SingleConnectionDatabaseManager, + DatabaseManager, SingleHostDiscovery, UrlReaders, } from '@backstage/backend-common'; @@ -34,8 +34,8 @@ function makeCreateEnv(config: Config) { root.info(`Created UrlReader ${reader}`); - const databaseManager = SingleConnectionDatabaseManager.fromConfig(config); const cacheManager = CacheManager.fromConfig(config); + const databaseManager = DatabaseManager.fromConfig(config); return (plugin: string): PluginEnvironment => { const logger = root.child({ type: 'plugin', plugin }); diff --git a/plugins/code-coverage-backend/src/service/CodeCoverageDatabase.test.ts b/plugins/code-coverage-backend/src/service/CodeCoverageDatabase.test.ts index 688eeb4ba2..fec2ea7afa 100644 --- a/plugins/code-coverage-backend/src/service/CodeCoverageDatabase.test.ts +++ b/plugins/code-coverage-backend/src/service/CodeCoverageDatabase.test.ts @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { SingleConnectionDatabaseManager } from '@backstage/backend-common'; +import { DatabaseManager } from '@backstage/backend-common'; import { stringifyEntityRef } from '@backstage/catalog-model'; import { ConfigReader } from '@backstage/config'; import { @@ -22,7 +22,7 @@ import { } from './CodeCoverageDatabase'; import { JsonCodeCoverage } from './types'; -const db = SingleConnectionDatabaseManager.fromConfig( +const db = DatabaseManager.fromConfig( new ConfigReader({ backend: { database: { diff --git a/plugins/code-coverage-backend/src/service/router.test.ts b/plugins/code-coverage-backend/src/service/router.test.ts index e18cd75c2f..340502d1e6 100644 --- a/plugins/code-coverage-backend/src/service/router.test.ts +++ b/plugins/code-coverage-backend/src/service/router.test.ts @@ -20,14 +20,14 @@ import { getVoidLogger, PluginDatabaseManager, PluginEndpointDiscovery, - SingleConnectionDatabaseManager, + DatabaseManager, UrlReaders, } from '@backstage/backend-common'; import { ConfigReader } from '@backstage/config'; import { createRouter } from './router'; function createDatabase(): PluginDatabaseManager { - return SingleConnectionDatabaseManager.fromConfig( + return DatabaseManager.fromConfig( new ConfigReader({ backend: { database: { diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.test.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.test.ts index 686dfc052f..5ed997bb25 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.test.ts @@ -14,17 +14,14 @@ * limitations under the License. */ -import { - getVoidLogger, - SingleConnectionDatabaseManager, -} from '@backstage/backend-common'; +import { getVoidLogger, DatabaseManager } from '@backstage/backend-common'; import { ConfigReader } from '@backstage/config'; import { DatabaseTaskStore } from './DatabaseTaskStore'; import { StorageTaskBroker, TaskAgent } from './StorageTaskBroker'; import { TaskSecrets, TaskSpec, DbTaskEventRow } from './types'; async function createStore(): Promise { - const manager = SingleConnectionDatabaseManager.fromConfig( + const manager = DatabaseManager.fromConfig( new ConfigReader({ backend: { database: { diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.test.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.test.ts index c6ad1a8105..f8343e7d3c 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.test.ts @@ -14,12 +14,9 @@ * limitations under the License. */ -import { - getVoidLogger, - SingleConnectionDatabaseManager, -} from '@backstage/backend-common'; -import { ConfigReader, JsonObject } from '@backstage/config'; import os from 'os'; +import { getVoidLogger, DatabaseManager } from '@backstage/backend-common'; +import { ConfigReader, JsonObject } from '@backstage/config'; import { createTemplateAction, TemplateActionRegistry } from '../actions'; import { RepoSpec } from '../actions/builtin/publish/util'; import { DatabaseTaskStore } from './DatabaseTaskStore'; @@ -27,7 +24,7 @@ import { StorageTaskBroker } from './StorageTaskBroker'; import { TaskWorker } from './TaskWorker'; async function createStore(): Promise { - const manager = SingleConnectionDatabaseManager.fromConfig( + const manager = DatabaseManager.fromConfig( new ConfigReader({ backend: { database: { diff --git a/plugins/scaffolder-backend/src/service/router.test.ts b/plugins/scaffolder-backend/src/service/router.test.ts index 9ca2eef9e6..9349e076fd 100644 --- a/plugins/scaffolder-backend/src/service/router.test.ts +++ b/plugins/scaffolder-backend/src/service/router.test.ts @@ -31,7 +31,7 @@ jest.doMock('fs-extra', () => ({ import { getVoidLogger, PluginDatabaseManager, - SingleConnectionDatabaseManager, + DatabaseManager, UrlReaders, } from '@backstage/backend-common'; import { CatalogApi } from '@backstage/catalog-client'; @@ -47,7 +47,7 @@ const createCatalogClient = (templates: any[] = []) => } as CatalogApi); function createDatabase(): PluginDatabaseManager { - return SingleConnectionDatabaseManager.fromConfig( + return DatabaseManager.fromConfig( new ConfigReader({ backend: { database: { From dc75c78cd776709b80ca82ce6924b449c5aabeca Mon Sep 17 00:00:00 2001 From: Minn Soe Date: Sat, 5 Jun 2021 23:15:44 +0100 Subject: [PATCH 21/93] refactor: split up database manager private methods Signed-off-by: Minn Soe --- packages/backend-common/api-report.md | 4 +- .../src/database/DatabaseManager.ts | 169 ++++++++++++------ .../backend-common/src/database/connection.ts | 4 +- 3 files changed, 115 insertions(+), 62 deletions(-) diff --git a/packages/backend-common/api-report.md b/packages/backend-common/api-report.md index d4d075004a..ed78157810 100644 --- a/packages/backend-common/api-report.md +++ b/packages/backend-common/api-report.md @@ -75,7 +75,7 @@ export interface CacheClient { export class CacheManager { forPlugin(pluginId: string): PluginCacheManager; static fromConfig(config: Config, options?: CacheManagerOptions): CacheManager; -} + } // @public (undocumented) export const coloredFormat: winston.Logform.Format; @@ -260,7 +260,7 @@ export class GitlabUrlReader implements UrlReader { export function loadBackendConfig(options: Options): Promise; // @public -export function normalizeConnection(connection: Knex.StaticConnectionConfig | JsonObject | string, client: string): Record; +export function normalizeConnection(connection: Knex.StaticConnectionConfig | JsonObject | string | undefined, client: string): Partial; // @public export function notFoundHandler(): RequestHandler; diff --git a/packages/backend-common/src/database/DatabaseManager.ts b/packages/backend-common/src/database/DatabaseManager.ts index 34ceea55e3..7513422452 100644 --- a/packages/backend-common/src/database/DatabaseManager.ts +++ b/packages/backend-common/src/database/DatabaseManager.ts @@ -15,7 +15,7 @@ */ import { Knex } from 'knex'; import { omit } from 'lodash'; -import { Config, ConfigReader } from '@backstage/config'; +import { Config, ConfigReader, JsonObject } from '@backstage/config'; import { createDatabaseClient, ensureDatabaseExists, @@ -24,6 +24,9 @@ import { } from './connection'; import { PluginDatabaseManager } from './types'; +/** + * Provides a config lookup path for a plugin's config block. + */ function pluginPath(pluginId: string): string { return `plugin.${pluginId}`; } @@ -56,8 +59,9 @@ export class DatabaseManager { /** * Generates a PluginDatabaseManager for consumption by plugins. * - * @param pluginId The plugin that the database manager should be created for. Plugin names should be unique - * as they are used to look up database config overrides under `backend.database.plugin`. + * @param pluginId The plugin that the database manager should be created for. Plugin names + * should be unique as they are used to look up database config overrides under + * `backend.database.plugin`. */ forPlugin(pluginId: string): PluginDatabaseManager { const _this = this; @@ -70,7 +74,7 @@ export class DatabaseManager { } /** - * Provides the canonical database name for a given pluginId. + * Provides the canonical database name for a given plugin. * * This method provides the effective database name which is determined using global * and plugin specific database config. If no explicit database name is configured, @@ -78,83 +82,132 @@ export class DatabaseManager { * 'backstage_plugin_'. * * @param pluginId Lookup the database name for given plugin + * @returns String representing the plugin's database name */ private getDatabaseName(pluginId: string): string { - const pluginConfig: Config = this.getConfigForPlugin(pluginId); + const connection = this.getConnectionConfig(pluginId); - // determine root sqlite config to pass through as this is a special case - const rootConnection = this.config.get('connection'); - const rootSqliteName = - typeof rootConnection === 'string' - ? rootConnection - : this.config.getOptionalString('connection.filename') ?? ':memory:'; - - const isSqlite = this.config.getString('client') === 'sqlite3'; + if (this.getClientType(pluginId).client === 'sqlite3') { + // sqlite database name should fallback to ':memory:' as a special case + return ( + (connection as Knex.Sqlite3ConnectionConfig)?.filename ?? ':memory:' + ); + } + // all other supported databases should fallback to an auto-prefixed name return ( - // attempt to lookup pg and mysql database name - pluginConfig.getOptionalString('connection.database') ?? - // attempt to lookup sqlite3 database file name - pluginConfig.getOptionalString('connection.filename') ?? - // if root is sqlite - attempt to use top level connection, fallback to :memory: - (isSqlite ? rootSqliteName : null) ?? - // generate a database name using prefix and pluginId + (connection as Knex.ConnectionConfig)?.database ?? `${this.prefix}${pluginId}` ); } /** - * Provides a base database connector config by merging different config sources. + * Provides the client type which should be used for a given plugin. * - * This method provides a baseConfig for a database connector without the target - * database's name property ('database', 'filename'). The client type is determined - * by plugin specific config which uses the default as the fallback. + * The client type is determined by plugin specific config if present. Otherwise the base + * client is used as the fallback. * - * If the client type is the same as the plugin or not specified, the global - * connection config will be extended with plugin specific config. - * - * @param pluginId The plugin that the database baseConfig should correspond to + * @param pluginId Plugin to get the client type for + * @returns Object with client type returned as `client` and boolean representing whether + * or not the client was overridden as `overridden` */ - private getConfigForPlugin(pluginId: string): Config { - const pluginConfig = this.config.getOptionalConfig(pluginPath(pluginId)); + private getClientType( + pluginId: string, + ): { + client: string; + overridden: boolean; + } { + const pluginClient = this.config.getOptionalString( + `${pluginPath(pluginId)}.client`, + ); const baseClient = this.config.getString('client'); - const client = pluginConfig?.getOptionalString('client') ?? baseClient; - - const baseConnection = normalizeConnection( - this.config.get('connection'), - baseClient, - ); - const connection = normalizeConnection( - pluginConfig?.getOptional('connection') ?? {}, + const client = pluginClient ?? baseClient; + return { client, - ); - - return new ConfigReader({ - client, - connection: { - // if same client type, extend original connection config without dbname config - ...(client === baseClient - ? omit(baseConnection, ['database', 'filename']) - : {}), - ...connection, - }, - }); + overridden: client !== baseClient, + }; } + /** + * Provides a Knex connection plugin config by combining base and plugin config. + * + * This method provides a baseConfig for a plugin database connector. If the client type + * has not been overridden, the global connection config will be included with plugin + * specific config as the base. Values from the plugin connection take precedence over the + * base. Base database name is omitted for all supported databases excluding SQLite. + */ + private getConnectionConfig( + pluginId: string, + ): Partial { + const { client, overridden } = this.getClientType(pluginId); + + let baseConnection = normalizeConnection( + this.config.get('connection'), + this.config.getString('client'), + ); + // As databases cannot be shared, the `database` property from the base connection + // is omitted. SQLite3's `filename` property is an exception as this is used as a + // directory elsewhere so we preserve `filename`. + baseConnection = omit(baseConnection, 'database'); + + // get and normalize optional plugin specific database connection + const connection = normalizeConnection( + this.config.getOptional(`${pluginPath(pluginId)}.connection`), + client, + ); + + return { + // include base connection if client type has not been overriden + ...(overridden ? {} : baseConnection), + ...connection, + }; + } + + /** + * Provides a Knex database config for a given plugin. + * + * This method provides a Knex configuration object along with the plugin's client type. + * + * @param pluginId The plugin that the database config should correspond with + */ + private getConfigForPlugin(pluginId: string): Knex.Config { + const { client } = this.getClientType(pluginId); + + return { + client, + connection: this.getConnectionConfig(pluginId), + }; + } + + /** + * Provides a partial Knex.Config database name override for a given plugin. + * + * @param pluginId Target plugin to get database name override + * @returns Partial Knex.Config with database name override + */ + private getDatabaseOverrides(pluginId: string): Knex.Config { + return createNameOverride( + this.getClientType(pluginId).client, + this.getDatabaseName(pluginId), + ); + } + + /** + * Provides a scoped Knex client for a plugin as per application config. + * + * @param pluginId Plugin to get a Knex client for + * @returns Promise which resolves to a scoped Knex database client for a plugin + */ private async getDatabase(pluginId: string): Promise { - const pluginConfig = this.getConfigForPlugin(pluginId); + const pluginConfig = new ConfigReader( + this.getConfigForPlugin(pluginId) as JsonObject, + ); await ensureDatabaseExists(pluginConfig, this.getDatabaseName(pluginId)); + return createDatabaseClient( pluginConfig, this.getDatabaseOverrides(pluginId), ); } - - private getDatabaseOverrides(pluginId: string): Knex.Config { - return createNameOverride( - this.getConfigForPlugin(pluginId).get('client'), - this.getDatabaseName(pluginId), - ); - } } diff --git a/packages/backend-common/src/database/connection.ts b/packages/backend-common/src/database/connection.ts index 005ed8819f..2f6d204839 100644 --- a/packages/backend-common/src/database/connection.ts +++ b/packages/backend-common/src/database/connection.ts @@ -121,9 +121,9 @@ export function parseConnectionString( * Normalizes a connection config or string into an object which can be passed to Knex. */ export function normalizeConnection( - connection: Knex.StaticConnectionConfig | JsonObject | string, + connection: Knex.StaticConnectionConfig | JsonObject | string | undefined, client: string, -): Record { +): Partial { if (typeof connection === 'undefined' || connection === null) { return {}; } From 214efffe3ac2f99dd08c37c3130fe4742511d271 Mon Sep 17 00:00:00 2001 From: Minn Soe Date: Sun, 6 Jun 2021 16:56:38 +0100 Subject: [PATCH 22/93] refactor: move database connector interface Signed-off-by: Minn Soe --- .../backend-common/src/database/connection.ts | 2 +- .../backend-common/src/database/connector.ts | 30 ---------------- packages/backend-common/src/database/mysql.ts | 2 +- .../backend-common/src/database/postgres.ts | 2 +- .../backend-common/src/database/sqlite3.ts | 2 +- packages/backend-common/src/database/types.ts | 35 +++++++++++++++++++ 6 files changed, 39 insertions(+), 34 deletions(-) delete mode 100644 packages/backend-common/src/database/connector.ts diff --git a/packages/backend-common/src/database/connection.ts b/packages/backend-common/src/database/connection.ts index 2f6d204839..2c2bba28ec 100644 --- a/packages/backend-common/src/database/connection.ts +++ b/packages/backend-common/src/database/connection.ts @@ -18,7 +18,7 @@ import { Config, JsonObject } from '@backstage/config'; import { InputError } from '@backstage/errors'; import knexFactory, { Knex } from 'knex'; import { mergeDatabaseConfig } from './config'; -import { DatabaseConnector } from './connector'; +import { DatabaseConnector } from './types'; import { mysqlConnector } from './mysql'; import { pgConnector } from './postgres'; diff --git a/packages/backend-common/src/database/connector.ts b/packages/backend-common/src/database/connector.ts deleted file mode 100644 index 0336510f19..0000000000 --- a/packages/backend-common/src/database/connector.ts +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright 2021 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import { Config } from '@backstage/config'; -import { Knex } from 'knex'; - -export interface DatabaseConnector { - createClient(dbConfig: Config, overrides?: Partial): Knex; - createNameOverride(name: string): Partial; - parseConnectionString( - connectionString: string, - client?: string, - ): Knex.StaticConnectionConfig; - ensureDatabaseExists?( - dbConfig: Config, - ...databases: Array - ): Promise; -} diff --git a/packages/backend-common/src/database/mysql.ts b/packages/backend-common/src/database/mysql.ts index beff5d9f82..0769ad64fc 100644 --- a/packages/backend-common/src/database/mysql.ts +++ b/packages/backend-common/src/database/mysql.ts @@ -18,7 +18,7 @@ import { Config } from '@backstage/config'; import { InputError } from '@backstage/errors'; import knexFactory, { Knex } from 'knex'; import { mergeDatabaseConfig } from './config'; -import { DatabaseConnector } from './connector'; +import { DatabaseConnector } from './types'; import yn from 'yn'; /** diff --git a/packages/backend-common/src/database/postgres.ts b/packages/backend-common/src/database/postgres.ts index 000b84460c..5662028113 100644 --- a/packages/backend-common/src/database/postgres.ts +++ b/packages/backend-common/src/database/postgres.ts @@ -17,7 +17,7 @@ import knexFactory, { Knex } from 'knex'; import { Config } from '@backstage/config'; import { mergeDatabaseConfig } from './config'; -import { DatabaseConnector } from './connector'; +import { DatabaseConnector } from './types'; /** * Creates a knex postgres database connection diff --git a/packages/backend-common/src/database/sqlite3.ts b/packages/backend-common/src/database/sqlite3.ts index 83f5f60699..2499415bef 100644 --- a/packages/backend-common/src/database/sqlite3.ts +++ b/packages/backend-common/src/database/sqlite3.ts @@ -19,7 +19,7 @@ import { ensureDirSync } from 'fs-extra'; import knexFactory, { Knex } from 'knex'; import path from 'path'; import { mergeDatabaseConfig } from './config'; -import { DatabaseConnector } from './connector'; +import { DatabaseConnector } from './types'; /** * Creates a knex sqlite3 database connection diff --git a/packages/backend-common/src/database/types.ts b/packages/backend-common/src/database/types.ts index bf3ceb2786..995f98f27d 100644 --- a/packages/backend-common/src/database/types.ts +++ b/packages/backend-common/src/database/types.ts @@ -14,6 +14,7 @@ * limitations under the License. */ +import { Config } from '@backstage/config'; import { Knex } from 'knex'; /** @@ -28,3 +29,37 @@ export interface PluginDatabaseManager { */ getClient(): Promise; } + +/** + * DatabaseConnector manages an underlying Knex database driver. + */ +export interface DatabaseConnector { + /** + * createClient provides an instance of a knex database connector. + */ + createClient(dbConfig: Config, overrides?: Partial): Knex; + /** + * createNameOverride provides a partial knex config sufficient to override a + * database name. + */ + createNameOverride(name: string): Partial; + /** + * parseConnectionString produces a knex connection config object representing + * a database connection string. + */ + parseConnectionString( + connectionString: string, + client?: string, + ): Knex.StaticConnectionConfig; + /** + * ensureDatabaseExists performs a side-effect to ensure database names passed in are + * present. + * + * Calling this function on databases which already exist should do nothing. + * Missing databases should be created if needed. + */ + ensureDatabaseExists?( + dbConfig: Config, + ...databases: Array + ): Promise; +} From f1a9108c53a9c1ff3cdf9a09e226dd556f6930b8 Mon Sep 17 00:00:00 2001 From: Minn Soe Date: Mon, 7 Jun 2021 13:07:24 +0100 Subject: [PATCH 23/93] refactor: add defaultNameOverride and reorganize Signed-off-by: Minn Soe --- packages/backend-common/api-report.md | 8 +++++ .../backend-common/src/database/connection.ts | 4 +-- .../connectors/defaultNameOverride.test.ts | 26 ++++++++++++++ .../connectors/defaultNameOverride.ts | 34 +++++++++++++++++++ .../src/database/connectors/index.ts | 18 ++++++++++ .../database/{ => connectors}/mysql.test.ts | 0 .../src/database/{ => connectors}/mysql.ts | 22 +++++------- .../{ => connectors}/postgres.test.ts | 0 .../src/database/{ => connectors}/postgres.ts | 16 +++------ .../database/{ => connectors}/sqlite3.test.ts | 0 .../src/database/{ => connectors}/sqlite3.ts | 21 ++++++++---- 11 files changed, 114 insertions(+), 35 deletions(-) create mode 100644 packages/backend-common/src/database/connectors/defaultNameOverride.test.ts create mode 100644 packages/backend-common/src/database/connectors/defaultNameOverride.ts create mode 100644 packages/backend-common/src/database/connectors/index.ts rename packages/backend-common/src/database/{ => connectors}/mysql.test.ts (100%) rename packages/backend-common/src/database/{ => connectors}/mysql.ts (93%) rename packages/backend-common/src/database/{ => connectors}/postgres.test.ts (100%) rename packages/backend-common/src/database/{ => connectors}/postgres.ts (93%) rename packages/backend-common/src/database/{ => connectors}/sqlite3.test.ts (100%) rename packages/backend-common/src/database/{ => connectors}/sqlite3.ts (90%) diff --git a/packages/backend-common/api-report.md b/packages/backend-common/api-report.md index ed78157810..1cf1f4f21b 100644 --- a/packages/backend-common/api-report.md +++ b/packages/backend-common/api-report.md @@ -104,6 +104,14 @@ export function createServiceBuilder(_module: NodeModule): ServiceBuilderImpl; // @public (undocumented) export function createStatusCheckRouter(options: StatusCheckRouterOptions): Promise; +// @public +export interface DatabaseConnector { + createClient(dbConfig: Config, overrides?: Partial): Knex; + createNameOverride(name: string): Partial; + ensureDatabaseExists?(dbConfig: Config, ...databases: Array): Promise; + parseConnectionString(connectionString: string, client?: string): Knex.StaticConnectionConfig; +} + // @public (undocumented) export class DatabaseManager { forPlugin(pluginId: string): PluginDatabaseManager; diff --git a/packages/backend-common/src/database/connection.ts b/packages/backend-common/src/database/connection.ts index 2c2bba28ec..46f040d9d0 100644 --- a/packages/backend-common/src/database/connection.ts +++ b/packages/backend-common/src/database/connection.ts @@ -20,9 +20,7 @@ import knexFactory, { Knex } from 'knex'; import { mergeDatabaseConfig } from './config'; import { DatabaseConnector } from './types'; -import { mysqlConnector } from './mysql'; -import { pgConnector } from './postgres'; -import { sqlite3Connector } from './sqlite3'; +import { mysqlConnector, pgConnector, sqlite3Connector } from './connectors'; type DatabaseClient = 'pg' | 'sqlite3' | 'mysql' | 'mysql2' | string; diff --git a/packages/backend-common/src/database/connectors/defaultNameOverride.test.ts b/packages/backend-common/src/database/connectors/defaultNameOverride.test.ts new file mode 100644 index 0000000000..b41736153a --- /dev/null +++ b/packages/backend-common/src/database/connectors/defaultNameOverride.test.ts @@ -0,0 +1,26 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import defaultNameOverride from './defaultNameOverride'; + +describe('defaultNameOverride()', () => { + it('returns a partial knex static connection config with database name', () => { + const testDatabaseName = 'testdatabase'; + expect(defaultNameOverride(testDatabaseName)).toHaveProperty( + 'connection.database', + testDatabaseName, + ); + }); +}); diff --git a/packages/backend-common/src/database/connectors/defaultNameOverride.ts b/packages/backend-common/src/database/connectors/defaultNameOverride.ts new file mode 100644 index 0000000000..6296010c76 --- /dev/null +++ b/packages/backend-common/src/database/connectors/defaultNameOverride.ts @@ -0,0 +1,34 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Knex } from 'knex'; + +/** + * Provides a partial knex config with database name override. + * + * Default override for knex database drivers which accept ConnectionConfig + * with `connection.database` as the database name field. + * + * @param name database name to get config override for + */ +export default function defaultNameOverride( + name: string, +): Partial { + return { + connection: { + database: name, + }, + }; +} diff --git a/packages/backend-common/src/database/connectors/index.ts b/packages/backend-common/src/database/connectors/index.ts new file mode 100644 index 0000000000..f314bb5004 --- /dev/null +++ b/packages/backend-common/src/database/connectors/index.ts @@ -0,0 +1,18 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +export * from './mysql'; +export * from './postgres'; +export * from './sqlite3'; diff --git a/packages/backend-common/src/database/mysql.test.ts b/packages/backend-common/src/database/connectors/mysql.test.ts similarity index 100% rename from packages/backend-common/src/database/mysql.test.ts rename to packages/backend-common/src/database/connectors/mysql.test.ts diff --git a/packages/backend-common/src/database/mysql.ts b/packages/backend-common/src/database/connectors/mysql.ts similarity index 93% rename from packages/backend-common/src/database/mysql.ts rename to packages/backend-common/src/database/connectors/mysql.ts index 0769ad64fc..60f1e09ce0 100644 --- a/packages/backend-common/src/database/mysql.ts +++ b/packages/backend-common/src/database/connectors/mysql.ts @@ -14,12 +14,14 @@ * limitations under the License. */ +import knexFactory, { Knex } from 'knex'; +import yn from 'yn'; + import { Config } from '@backstage/config'; import { InputError } from '@backstage/errors'; -import knexFactory, { Knex } from 'knex'; -import { mergeDatabaseConfig } from './config'; -import { DatabaseConnector } from './types'; -import yn from 'yn'; +import { mergeDatabaseConfig } from '../config'; +import { DatabaseConnector } from '../types'; +import defaultNameOverride from './defaultNameOverride'; /** * Creates a knex mysql database connection @@ -161,22 +163,14 @@ export async function ensureMysqlDatabaseExists( } } -export function createMysqlNameOverride(name: string): Partial { - return { - connection: { - database: name, - }, - }; -} - /** - * MySql database connector. + * MySQL database connector. * * Exposes database connector functionality via an immutable object. */ export const mysqlConnector: DatabaseConnector = Object.freeze({ createClient: createMysqlDatabaseClient, ensureDatabaseExists: ensureMysqlDatabaseExists, - createNameOverride: createMysqlNameOverride, + createNameOverride: defaultNameOverride, parseConnectionString: parseMysqlConnectionString, }); diff --git a/packages/backend-common/src/database/postgres.test.ts b/packages/backend-common/src/database/connectors/postgres.test.ts similarity index 100% rename from packages/backend-common/src/database/postgres.test.ts rename to packages/backend-common/src/database/connectors/postgres.test.ts diff --git a/packages/backend-common/src/database/postgres.ts b/packages/backend-common/src/database/connectors/postgres.ts similarity index 93% rename from packages/backend-common/src/database/postgres.ts rename to packages/backend-common/src/database/connectors/postgres.ts index 5662028113..011e40579b 100644 --- a/packages/backend-common/src/database/postgres.ts +++ b/packages/backend-common/src/database/connectors/postgres.ts @@ -15,9 +15,11 @@ */ import knexFactory, { Knex } from 'knex'; + import { Config } from '@backstage/config'; -import { mergeDatabaseConfig } from './config'; -import { DatabaseConnector } from './types'; +import { mergeDatabaseConfig } from '../config'; +import { DatabaseConnector } from '../types'; +import defaultNameOverride from './defaultNameOverride'; /** * Creates a knex postgres database connection @@ -133,14 +135,6 @@ export async function ensurePgDatabaseExists( } } -export function createPgNameOverride(name: string): Partial { - return { - connection: { - database: name, - }, - }; -} - /** * PostgreSQL database connector. * @@ -149,6 +143,6 @@ export function createPgNameOverride(name: string): Partial { export const pgConnector: DatabaseConnector = Object.freeze({ createClient: createPgDatabaseClient, ensureDatabaseExists: ensurePgDatabaseExists, - createNameOverride: createPgNameOverride, + createNameOverride: defaultNameOverride, parseConnectionString: parsePgConnectionString, }); diff --git a/packages/backend-common/src/database/sqlite3.test.ts b/packages/backend-common/src/database/connectors/sqlite3.test.ts similarity index 100% rename from packages/backend-common/src/database/sqlite3.test.ts rename to packages/backend-common/src/database/connectors/sqlite3.test.ts diff --git a/packages/backend-common/src/database/sqlite3.ts b/packages/backend-common/src/database/connectors/sqlite3.ts similarity index 90% rename from packages/backend-common/src/database/sqlite3.ts rename to packages/backend-common/src/database/connectors/sqlite3.ts index 2499415bef..c9e86c80da 100644 --- a/packages/backend-common/src/database/sqlite3.ts +++ b/packages/backend-common/src/database/connectors/sqlite3.ts @@ -13,16 +13,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +import path from 'path'; -import { Config } from '@backstage/config'; import { ensureDirSync } from 'fs-extra'; import knexFactory, { Knex } from 'knex'; -import path from 'path'; -import { mergeDatabaseConfig } from './config'; -import { DatabaseConnector } from './types'; + +import { Config } from '@backstage/config'; +import { mergeDatabaseConfig } from '../config'; +import { DatabaseConnector } from '../types'; /** - * Creates a knex sqlite3 database connection + * Creates a knex SQLite3 database connection * * @param dbConfig The database config * @param overrides Additional options to merge with the config @@ -55,7 +56,7 @@ export function createSqliteDatabaseClient( } /** - * Builds a knex sqlite3 connection config + * Builds a knex SQLite3 connection config * * @param dbConfig The database config * @param overrides Additional options to merge with the config @@ -101,12 +102,18 @@ export function buildSqliteDatabaseConfig( return config; } +/** + * Provides a partial knex SQLite3 config to override database name. + */ export function createSqliteNameOverride(name: string): Partial { return { connection: parseSqliteConnectionString(name), }; } +/** + * Produces a partial knex SQLite3 connection config with database name. + */ export function parseSqliteConnectionString( name: string, ): Knex.Sqlite3ConnectionConfig { @@ -116,7 +123,7 @@ export function parseSqliteConnectionString( } /** - * Sqlite3 database connector. + * SQLite3 database connector. * * Exposes database connector functionality via an immutable object. */ From 946db4caf65d4693cc4ed11f740b123095284a4e Mon Sep 17 00:00:00 2001 From: Minn Soe Date: Thu, 10 Jun 2021 11:11:21 +0100 Subject: [PATCH 24/93] docs: expand changeset scope `@backstage/backend-test-utils` has been updated to use a plugin name which starts with an alphabetical character, changeset now includes this package. Signed-off-by: Minn Soe --- .changeset/five-donkeys-brake.md | 1 + 1 file changed, 1 insertion(+) diff --git a/.changeset/five-donkeys-brake.md b/.changeset/five-donkeys-brake.md index 8a6523bb61..b0bb635876 100644 --- a/.changeset/five-donkeys-brake.md +++ b/.changeset/five-donkeys-brake.md @@ -1,6 +1,7 @@ --- '@backstage/backend-common': minor '@backstage/create-app': minor +'@backstage/backend-test-utils': minor --- Deprecates `SingleConnectionDatabaseManager` and provides an API compatible database From a89ec167e1d23ec24bf0cffb178cc535a52818d4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 11 Jun 2021 04:10:19 +0000 Subject: [PATCH 25/93] chore(deps): bump leasot from 11.5.0 to 12.0.0 Bumps [leasot](https://github.com/pgilad/leasot) from 11.5.0 to 12.0.0. - [Release notes](https://github.com/pgilad/leasot/releases) - [Commits](https://github.com/pgilad/leasot/compare/v11.5.0...v12.0.0) --- updated-dependencies: - dependency-name: leasot dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- plugins/todo-backend/package.json | 2 +- yarn.lock | 56 ++++++++++++------------------- 2 files changed, 23 insertions(+), 35 deletions(-) diff --git a/plugins/todo-backend/package.json b/plugins/todo-backend/package.json index a49bd8e408..309846c46f 100644 --- a/plugins/todo-backend/package.json +++ b/plugins/todo-backend/package.json @@ -34,7 +34,7 @@ "cross-fetch": "^3.0.6", "express": "^4.17.1", "express-promise-router": "^4.1.0", - "leasot": "^11.5.0", + "leasot": "^12.0.0", "winston": "^3.2.1", "yn": "^4.0.0" }, diff --git a/yarn.lock b/yarn.lock index b7c74d889a..c64b81e60b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1347,7 +1347,7 @@ to-fast-properties "^2.0.0" "@backstage/catalog-model@^0.7.4": - version "0.8.1" + version "0.8.2" dependencies: "@backstage/config" "^0.1.5" "@backstage/errors" "^0.1.1" @@ -1361,7 +1361,7 @@ yup "^0.29.3" "@backstage/catalog-model@^0.7.9": - version "0.8.1" + version "0.8.2" dependencies: "@backstage/config" "^0.1.5" "@backstage/errors" "^0.1.1" @@ -1391,15 +1391,15 @@ react-use "^17.2.4" "@backstage/plugin-catalog@^0.5.1": - version "0.6.1" + version "0.6.2" dependencies: - "@backstage/catalog-client" "^0.3.12" - "@backstage/catalog-model" "^0.8.1" + "@backstage/catalog-client" "^0.3.13" + "@backstage/catalog-model" "^0.8.2" "@backstage/core" "^0.7.12" "@backstage/errors" "^0.1.1" - "@backstage/integration" "^0.5.5" - "@backstage/integration-react" "^0.1.2" - "@backstage/plugin-catalog-react" "^0.2.1" + "@backstage/integration" "^0.5.6" + "@backstage/integration-react" "^0.1.3" + "@backstage/plugin-catalog-react" "^0.2.2" "@backstage/theme" "^0.2.8" "@material-ui/core" "^4.11.0" "@material-ui/icons" "^4.9.1" @@ -9847,15 +9847,15 @@ commander@^5.0.0, commander@^5.1.0: resolved "https://registry.npmjs.org/commander/-/commander-5.1.0.tgz#46abbd1652f8e059bddaef99bbdcb2ad9cf179ae" integrity sha512-P0CysNDQ7rtVw4QIQtm+MRxV66vKFSvlsQvGYXZWR3qFU0jlMKHZZZgw8e+8DSah4UDKMqnknRDQz+xuQXQ/Zg== -commander@^6.1.0, commander@^6.2.1: +commander@^6.1.0: version "6.2.1" resolved "https://registry.npmjs.org/commander/-/commander-6.2.1.tgz#0792eb682dfbc325999bb2b84fddddba110ac73c" integrity sha512-U7VdrJFnJgo4xjrHpTzu0yrHPGImdsmD95ZlgYSEajAn2JKzDhDTPG9kBTefmObL2w/ngeZnilk+OV9CG3d7UA== -commander@^7.1.0: - version "7.1.0" - resolved "https://registry.npmjs.org/commander/-/commander-7.1.0.tgz#f2eaecf131f10e36e07d894698226e36ae0eb5ff" - integrity sha512-pRxBna3MJe6HKnBGsDyMv8ETbptw3axEdYHoqNh7gu5oDcew8fs0xnivZGm06Ogk8zGAJ9VX+OPEr2GXEQK4dg== +commander@^7.1.0, commander@^7.2.0: + version "7.2.0" + resolved "https://registry.npmjs.org/commander/-/commander-7.2.0.tgz#a36cb57d0b501ce108e4d20559a150a391d97ab7" + integrity sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw== common-tags@1.8.0, common-tags@^1.8.0: version "1.8.0" @@ -14009,7 +14009,7 @@ globby@11.0.1: merge2 "^1.3.0" slash "^3.0.0" -globby@11.0.3, globby@^11.0.3: +globby@11.0.3, globby@^11.0.0, globby@^11.0.1, globby@^11.0.2, globby@^11.0.3: version "11.0.3" resolved "https://registry.npmjs.org/globby/-/globby-11.0.3.tgz#9b1f0cb523e171dd1ad8c7b2a9fb4b644b9593cb" integrity sha512-ffdmosjA807y7+lA1NM0jELARVmYul/715xiILEjo3hBLPTcirgQNnXECn5g3mtR8TOLCVbkfua1Hpen25/Xcg== @@ -14048,18 +14048,6 @@ globby@^10.0.1: merge2 "^1.2.3" slash "^3.0.0" -globby@^11.0.0, globby@^11.0.1, globby@^11.0.2: - version "11.0.2" - resolved "https://registry.npmjs.org/globby/-/globby-11.0.2.tgz#1af538b766a3b540ebfb58a32b2e2d5897321d83" - integrity sha512-2ZThXDvvV8fYFRVIxnrMQBipZQDr7MxKAmQK1vujaj9/7eF0efG7BPUKJ7jP7G5SLF37xKDXvO4S/KKLj/Z0og== - dependencies: - array-union "^2.1.0" - dir-glob "^3.0.1" - fast-glob "^3.1.1" - ignore "^5.1.4" - merge2 "^1.3.0" - slash "^3.0.0" - globby@^6.1.0: version "6.1.0" resolved "https://registry.npmjs.org/globby/-/globby-6.1.0.tgz#f5a6d70e8395e21c858fb0489d64df02424d506c" @@ -17312,20 +17300,20 @@ ldapjs@^2.2.0: vasync "^2.2.0" verror "^1.8.1" -leasot@^11.5.0: - version "11.5.0" - resolved "https://registry.npmjs.org/leasot/-/leasot-11.5.0.tgz#a99eb4479618c9d2ea442a32ee006e5b9da4844d" - integrity sha512-L08QKlmofYIRs5gfOmhOtbEJUu6U/zFGvYpboPq34yAHQ2Oc/QznOw62noe29yRJLiV/XnIDS8vO2um1e1sikA== +leasot@^12.0.0: + version "12.0.0" + resolved "https://registry.npmjs.org/leasot/-/leasot-12.0.0.tgz#78c5df2c941c7285374c8d992866e22163241b22" + integrity sha512-TMe3cJTRUMpXsOFNXCig5U84wM44y84vawkl2fC7iAJif88l/b7BtTt49VrkMsivlxlqHYVu5PjuxB9sRQf39w== dependencies: async "^3.2.0" chalk "^4.1.0" - commander "^6.2.1" + commander "^7.2.0" eol "^0.9.1" get-stdin "^8.0.0" - globby "^11.0.1" + globby "^11.0.3" json2xml "^0.1.3" - lodash "^4.17.20" - log-symbols "^4.0.0" + lodash "^4.17.21" + log-symbols "^4.1.0" strip-ansi "^6.0.0" text-table "^0.2.0" From 86fc1c39301ede302485da28d7ab39277a8bdcea Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Fri, 11 Jun 2021 10:35:01 +0200 Subject: [PATCH 26/93] Carve out exception for external SVGs Signed-off-by: Eric Peterson --- .../reader/transformers/addBaseUrl.test.ts | 65 ++++++++++++++++++- .../src/reader/transformers/addBaseUrl.ts | 19 +++++- 2 files changed, 80 insertions(+), 4 deletions(-) diff --git a/plugins/techdocs/src/reader/transformers/addBaseUrl.test.ts b/plugins/techdocs/src/reader/transformers/addBaseUrl.test.ts index 2b17d04144..5a3205f886 100644 --- a/plugins/techdocs/src/reader/transformers/addBaseUrl.test.ts +++ b/plugins/techdocs/src/reader/transformers/addBaseUrl.test.ts @@ -19,12 +19,15 @@ import { addBaseUrl } from '../transformers'; import { TechDocsStorageApi } from '../../api'; const DOC_STORAGE_URL = 'https://example-host.storage.googleapis.com'; +const API_ORIGIN_URL = 'https://backstage.example.com/api/techdocs'; const techdocsStorageApi: TechDocsStorageApi = { - getBaseUrl: jest.fn(() => Promise.resolve(DOC_STORAGE_URL)), + getBaseUrl: jest.fn(o => + Promise.resolve(new URL(o, DOC_STORAGE_URL).toString()), + ), getEntityDocs: () => new Promise(resolve => resolve('yes!')), syncEntityDocs: () => new Promise(resolve => resolve(true)), - getApiOrigin: jest.fn(), + getApiOrigin: jest.fn(() => new Promise(resolve => resolve(API_ORIGIN_URL))), getBuilder: jest.fn(), getStorageUrl: jest.fn(), }; @@ -96,7 +99,7 @@ describe('addBaseUrl', () => { ); }); - it('transforms svg img src to data uri', async () => { + it('inlines svg img src to data uri', async () => { const svgContent = ''; const expectedSrc = `data:image/svg+xml;base64,${Buffer.from( svgContent, @@ -125,4 +128,60 @@ describe('addBaseUrl', () => { }); }); }); + + it('inlines absolute url svgs pointed at our backend', async () => { + const svgContent = ''; + const expectedSrc = `data:image/svg+xml;base64,${Buffer.from( + svgContent, + ).toString('base64')}`; + + (global.fetch as jest.Mock).mockReturnValue({ + text: jest.fn().mockResolvedValue(svgContent), + }); + + const root = createTestShadowDom( + ``, + { + preTransformers: [ + addBaseUrl({ + techdocsStorageApi, + entityId: mockEntityId, + path: '', + }), + ], + postTransformers: [], + }, + ); + + await new Promise(done => { + process.nextTick(() => { + const actualSrc = root.getElementById('x')?.getAttribute('src'); + expect(expectedSrc).toEqual(actualSrc); + done(); + }); + }); + }); + + it('does not inline external svgs', async () => { + const expectedSrc = 'https://example.com/test.svg'; + const root = createTestShadowDom(``, { + preTransformers: [ + addBaseUrl({ + techdocsStorageApi, + entityId: mockEntityId, + path: '', + }), + ], + postTransformers: [], + }); + + await new Promise(done => { + process.nextTick(() => { + const actualElem = root.getElementById('x'); + expect(actualElem?.getAttribute('src')).toEqual(expectedSrc); + expect(actualElem?.getAttribute('alt')).toEqual(null); + done(); + }); + }); + }); }); diff --git a/plugins/techdocs/src/reader/transformers/addBaseUrl.ts b/plugins/techdocs/src/reader/transformers/addBaseUrl.ts index 98db162b46..dc4eecde16 100644 --- a/plugins/techdocs/src/reader/transformers/addBaseUrl.ts +++ b/plugins/techdocs/src/reader/transformers/addBaseUrl.ts @@ -23,6 +23,22 @@ type AddBaseUrlOptions = { path: string; }; +/** + * TechDocs backend serves SVGs with text/plain content-type for security. This + * helper determines if an SVG is being loaded from the backend, and thus needs + * inlining to be displayed properly. + */ +const isSvgNeedingInlining = ( + attrName: string, + attrVal: string, + apiOrigin: string, +) => { + const isSrcToSvg = attrName === 'src' && attrVal.endsWith('.svg'); + const isRelativeUrl = !attrVal.match(/^([a-z]*:)?\/\//i); + const pointsToOurBackend = attrVal.startsWith(apiOrigin); + return isSrcToSvg && (isRelativeUrl || pointsToOurBackend); +}; + export const addBaseUrl = ({ techdocsStorageApi, entityId, @@ -45,7 +61,8 @@ export const addBaseUrl = ({ entityId, path, ); - if (attributeName === 'src' && elemAttribute.endsWith('.svg')) { + const apiOrigin = await techdocsStorageApi.getApiOrigin(); + if (isSvgNeedingInlining(attributeName, elemAttribute, apiOrigin)) { try { const svg = await fetch(newValue); const svgContent = await svg.text(); From 9b57fda8b2191b2222242fecdab77c33931d0575 Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Fri, 11 Jun 2021 10:41:19 +0200 Subject: [PATCH 27/93] Changeset Signed-off-by: Eric Peterson --- .changeset/techdocs-a-primeira-vez.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changeset/techdocs-a-primeira-vez.md diff --git a/.changeset/techdocs-a-primeira-vez.md b/.changeset/techdocs-a-primeira-vez.md new file mode 100644 index 0000000000..efc626091c --- /dev/null +++ b/.changeset/techdocs-a-primeira-vez.md @@ -0,0 +1,6 @@ +--- +'@backstage/plugin-techdocs': patch +--- + +Fixes a bug that could prevent some externally hosted images (like icons or +build badges) from rendering within TechDocs documentation. From 873116e5df1f1a445c1213132cfc109e02dc9e90 Mon Sep 17 00:00:00 2001 From: Oliver Sand Date: Fri, 11 Jun 2021 12:37:54 +0200 Subject: [PATCH 28/93] Fix a react warning in `` `entityRef` should only be passed conditionally if component is present, otherwise React logs a warning/error to console. Signed-off-by: Oliver Sand --- .changeset/nice-spoons-try.md | 5 +++++ .../EntityListComponent/EntityListComponent.tsx | 10 +++++++--- 2 files changed, 12 insertions(+), 3 deletions(-) create mode 100644 .changeset/nice-spoons-try.md diff --git a/.changeset/nice-spoons-try.md b/.changeset/nice-spoons-try.md new file mode 100644 index 0000000000..c0992fa05e --- /dev/null +++ b/.changeset/nice-spoons-try.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-import': patch +--- + +Fix a react warning in ``. diff --git a/plugins/catalog-import/src/components/EntityListComponent/EntityListComponent.tsx b/plugins/catalog-import/src/components/EntityListComponent/EntityListComponent.tsx index bdeb483df4..64290d223e 100644 --- a/plugins/catalog-import/src/components/EntityListComponent/EntityListComponent.tsx +++ b/plugins/catalog-import/src/components/EntityListComponent/EntityListComponent.tsx @@ -146,11 +146,15 @@ export const EntityListComponent = ({ {sortEntities(r.entities).map(entity => ( {getEntityIcon(entity)} From e2048aec9ddb6b1cf7a3ebbe8da10985bd0e695a Mon Sep 17 00:00:00 2001 From: Fabian Hippmann Date: Sat, 12 Jun 2021 01:26:00 +0200 Subject: [PATCH 29/93] Add MoonShiner to the adopters list Signed-off-by: Fabian Hippmann --- ADOPTERS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/ADOPTERS.md b/ADOPTERS.md index 31f384e4e6..ceb6f20d31 100644 --- a/ADOPTERS.md +++ b/ADOPTERS.md @@ -30,3 +30,4 @@ | [Netflix](https://www.netflix.com/) | [bleathem](https://github.com/bleathem) | Our Backstage implementation will be the front door to a unified experience connecting our internal platform products across important workflows with integrated knowledge and support. | | [b.well](https://www.icanbwell.com/) | [Jacob Rosales](https://github.com/jrosales) | Foundation for our engineering portal and cloud insights. | | [PagerDuty](https://www.pagerduty.com/) | [Mark Shaw](https://github.com/markshawtoronto) | Developer portal, initially focused on software templates and tech-docs. | +| [MoonShiner](https://moonshiner.at) | [Fabian Hippmann](https://github.com/FabianHippmann) | Developer portal - helps us keep track of our customer projects, onboard new developers & improve our development process 🌕🚀🧑‍🚀 | From 4bd96b110f752481c6357c6f06617578b83fa7f9 Mon Sep 17 00:00:00 2001 From: Vitor Capretz Date: Sat, 12 Jun 2021 13:42:40 +0200 Subject: [PATCH 30/93] Replace timeago.js in favor of luxon in Sentry plugin Signed-off-by: Vitor Capretz --- plugins/sentry/package.json | 5 +++-- .../SentryIssuesTable/SentryIssuesTable.tsx | 7 ++++--- yarn.lock | 15 ++++++++++----- 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/plugins/sentry/package.json b/plugins/sentry/package.json index efaac81188..72404009e2 100644 --- a/plugins/sentry/package.json +++ b/plugins/sentry/package.json @@ -38,12 +38,12 @@ "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", "@material-ui/lab": "4.0.0-alpha.45", + "luxon": "^1.27.0", "react": "^16.13.1", "react-dom": "^16.13.1", "react-router": "6.0.0-beta.0", "react-sparklines": "^1.7.0", - "react-use": "^17.2.4", - "timeago.js": "^4.0.2" + "react-use": "^17.2.4" }, "devDependencies": { "@backstage/cli": "^0.7.0", @@ -53,6 +53,7 @@ "@testing-library/react": "^11.2.5", "@testing-library/user-event": "^13.1.8", "@types/jest": "^26.0.7", + "@types/luxon": "^1.27.0", "@types/node": "^14.14.32", "@types/react": "^16.9", "cross-fetch": "^3.0.6", diff --git a/plugins/sentry/src/components/SentryIssuesTable/SentryIssuesTable.tsx b/plugins/sentry/src/components/SentryIssuesTable/SentryIssuesTable.tsx index 72081bf2da..ea040f4130 100644 --- a/plugins/sentry/src/components/SentryIssuesTable/SentryIssuesTable.tsx +++ b/plugins/sentry/src/components/SentryIssuesTable/SentryIssuesTable.tsx @@ -17,7 +17,7 @@ import React from 'react'; import { Table, TableColumn } from '@backstage/core'; import { SentryIssue } from '../../api'; -import { format } from 'timeago.js'; +import { DateTime } from 'luxon'; import { ErrorCell } from '../ErrorCell/ErrorCell'; import { ErrorGraph } from '../ErrorGraph/ErrorGraph'; @@ -35,7 +35,8 @@ const columns: TableColumn[] = [ field: 'firstSeen', render: data => { const { firstSeen } = data as SentryIssue; - return format(firstSeen); + + return DateTime.fromISO(firstSeen).toRelative({ locale: 'en' }); }, }, { @@ -43,7 +44,7 @@ const columns: TableColumn[] = [ field: 'lastSeen', render: data => { const { lastSeen } = data as SentryIssue; - return format(lastSeen); + return DateTime.fromISO(lastSeen).toRelative({ locale: 'en' }); }, }, { diff --git a/yarn.lock b/yarn.lock index af81214be9..d7be55db7d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6039,6 +6039,11 @@ resolved "https://registry.npmjs.org/@types/luxon/-/luxon-1.26.5.tgz#843fb705e16e4d2a90847a351b799ea9d879859e" integrity sha512-XeQxxRMyJi1znfzHw4CGDLyup/raj84SnjjkI2fDootZPGlB0yqtvlvEIAmzHDa5wiEI5JJevZOWxpcofsaV+A== +"@types/luxon@^1.27.0": + version "1.27.0" + resolved "https://registry.npmjs.org/@types/luxon/-/luxon-1.27.0.tgz#1e3b5a7f8ca6944349c43498b4442b742c71ab0b" + integrity sha512-rr2lNXsErnA/ARtgFn46NtQjUa66cuwZYeo/2K7oqqxhJErhXgHBPyNKCo+pfOC3L7HFwtao8ebViiU9h4iAxA== + "@types/markdown-to-jsx@^6.11.0": version "6.11.2" resolved "https://registry.npmjs.org/@types/markdown-to-jsx/-/markdown-to-jsx-6.11.2.tgz#05d1aaffbf15be7be12c70535fa4fed65cc7c64f" @@ -17952,6 +17957,11 @@ luxon@^1.26.0: resolved "https://registry.npmjs.org/luxon/-/luxon-1.26.0.tgz#d3692361fda51473948252061d0f8561df02b578" integrity sha512-+V5QIQ5f6CDXQpWNICELwjwuHdqeJM1UenlZWx5ujcRMc9venvluCjFb4t5NYLhb6IhkbMVOxzVuOqkgMxee2A== +luxon@^1.27.0: + version "1.27.0" + resolved "https://registry.npmjs.org/luxon/-/luxon-1.27.0.tgz#ae10c69113d85dab8f15f5e8390d0cbeddf4f00f" + integrity sha512-VKsFsPggTA0DvnxtJdiExAucKdAnwbCCNlMM5ENvHlxubqWd0xhZcdb4XgZ7QFNhaRhilXCFxHuoObP5BNA4PA== + lz-string@^1.4.4: version "1.4.4" resolved "https://registry.npmjs.org/lz-string/-/lz-string-1.4.4.tgz#c0d8eaf36059f705796e1e344811cf4c498d3a26" @@ -25186,11 +25196,6 @@ tildify@2.0.0: resolved "https://registry.npmjs.org/tildify/-/tildify-2.0.0.tgz#f205f3674d677ce698b7067a99e949ce03b4754a" integrity sha512-Cc+OraorugtXNfs50hU9KS369rFXCfgGLpfCfvlc+Ud5u6VWmUQsOAa9HbTvheQdYnrdJqqv1e5oIqXppMYnSw== -timeago.js@^4.0.2: - version "4.0.2" - resolved "https://registry.npmjs.org/timeago.js/-/timeago.js-4.0.2.tgz#724e8c8833e3490676c7bb0a75f5daf20e558028" - integrity sha512-a7wPxPdVlQL7lqvitHGGRsofhdwtkoSXPGATFuSOA2i1ZNQEPLrGnj68vOp2sOJTCFAQVXPeNMX/GctBaO9L2w== - timers-browserify@^2.0.4: version "2.0.11" resolved "https://registry.npmjs.org/timers-browserify/-/timers-browserify-2.0.11.tgz#800b1f3eee272e5bc53ee465a04d0e804c31211f" From eeed844d7845465f435719c783e8f9a73ea1b9b2 Mon Sep 17 00:00:00 2001 From: Vitor Capretz Date: Sat, 12 Jun 2021 14:38:31 +0200 Subject: [PATCH 31/93] remove moment as step 1 of migration to lexon Signed-off-by: Vitor Capretz --- plugins/circleci/package.json | 1 - .../lib/ActionOutput/ActionOutput.tsx | 10 +++------- yarn.lock | 2 +- 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/plugins/circleci/package.json b/plugins/circleci/package.json index 3091996034..a7cd5f6b84 100644 --- a/plugins/circleci/package.json +++ b/plugins/circleci/package.json @@ -41,7 +41,6 @@ "circleci-api": "^4.0.0", "dayjs": "^1.9.4", "lodash": "^4.17.15", - "moment": "^2.25.3", "react": "^16.13.1", "react-dom": "^16.13.1", "react-lazylog": "^4.5.2", diff --git a/plugins/circleci/src/components/BuildWithStepsPage/lib/ActionOutput/ActionOutput.tsx b/plugins/circleci/src/components/BuildWithStepsPage/lib/ActionOutput/ActionOutput.tsx index e446c002f6..dcc5e36e94 100644 --- a/plugins/circleci/src/components/BuildWithStepsPage/lib/ActionOutput/ActionOutput.tsx +++ b/plugins/circleci/src/components/BuildWithStepsPage/lib/ActionOutput/ActionOutput.tsx @@ -24,11 +24,10 @@ import { import { makeStyles } from '@material-ui/core/styles'; import ExpandMoreIcon from '@material-ui/icons/ExpandMore'; import { BuildStepAction } from 'circleci-api'; -import moment from 'moment'; import React, { Suspense, useEffect, useState } from 'react'; +import { durationHumanized } from '../../../../util'; const LazyLog = React.lazy(() => import('react-lazylog/build/LazyLog')); -moment.relativeTimeThreshold('ss', 0); const useStyles = makeStyles({ accordionDetails: { padding: 0, @@ -66,11 +65,8 @@ export const ActionOutput = ({ }); }, [url]); - const timeElapsed = moment - .duration( - moment(action.end_time || moment()).diff(moment(action.start_time)), - ) - .humanize(); + const timeElapsed = durationHumanized(action.start_time, action.end_time); + return ( Date: Sat, 12 Jun 2021 14:39:20 +0200 Subject: [PATCH 32/93] Create changeset Signed-off-by: Vitor Capretz --- .changeset/mean-moose-sneeze.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/mean-moose-sneeze.md diff --git a/.changeset/mean-moose-sneeze.md b/.changeset/mean-moose-sneeze.md new file mode 100644 index 0000000000..42982c802c --- /dev/null +++ b/.changeset/mean-moose-sneeze.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-circleci': patch +--- + +Remove moment as part 1 of migration to lexon From b861c082b60b2eb1a61efb90611b371d0409ee87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mathias=20A=CC=8Ahsberg?= Date: Sat, 12 Jun 2021 14:41:48 +0000 Subject: [PATCH 33/93] Support jenkins build details for branches that contains slashes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Mathias Åhsberg --- .changeset/honest-pianos-smell.md | 5 +++++ .../src/components/BuildWithStepsPage/BuildWithStepsPage.tsx | 4 +++- .../src/components/BuildsPage/lib/CITable/CITable.tsx | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) create mode 100644 .changeset/honest-pianos-smell.md diff --git a/.changeset/honest-pianos-smell.md b/.changeset/honest-pianos-smell.md new file mode 100644 index 0000000000..56b87ce72d --- /dev/null +++ b/.changeset/honest-pianos-smell.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-jenkins': patch +--- + +Support showing build details for branches with slashes in their names diff --git a/plugins/jenkins/src/components/BuildWithStepsPage/BuildWithStepsPage.tsx b/plugins/jenkins/src/components/BuildWithStepsPage/BuildWithStepsPage.tsx index 1b7ccb8121..192f84458d 100644 --- a/plugins/jenkins/src/components/BuildWithStepsPage/BuildWithStepsPage.tsx +++ b/plugins/jenkins/src/components/BuildWithStepsPage/BuildWithStepsPage.tsx @@ -50,7 +50,9 @@ const BuildWithStepsView = () => { const projectName = useProjectSlugFromEntity(); const { branch, buildNumber } = useRouteRefParams(buildRouteRef); const classes = useStyles(); - const buildPath = `${projectName}/${branch}/${buildNumber}`; + const buildPath = `${projectName}/${encodeURIComponent( + branch, + )}/${buildNumber}`; const [{ value }] = useBuildWithSteps(buildPath); return ( diff --git a/plugins/jenkins/src/components/BuildsPage/lib/CITable/CITable.tsx b/plugins/jenkins/src/components/BuildsPage/lib/CITable/CITable.tsx index 57ffe48649..12d221d928 100644 --- a/plugins/jenkins/src/components/BuildsPage/lib/CITable/CITable.tsx +++ b/plugins/jenkins/src/components/BuildsPage/lib/CITable/CITable.tsx @@ -118,7 +118,7 @@ const generatedColumns: TableColumn[] = [ From 4ca32282688a816d7fe6ec518a296d565026ec99 Mon Sep 17 00:00:00 2001 From: Dominik Henneke Date: Mon, 14 Jun 2021 10:38:24 +0200 Subject: [PATCH 34/93] Migrate from the `command-exists-promise` dependency to `command-exists` Signed-off-by: Dominik Henneke --- .changeset/thick-donkeys-carry.md | 5 +++++ plugins/scaffolder-backend/package.json | 2 +- .../src/scaffolder/stages/templater/cookiecutter.test.ts | 2 +- .../src/scaffolder/stages/templater/cookiecutter.ts | 2 +- yarn.lock | 5 ----- 5 files changed, 8 insertions(+), 8 deletions(-) create mode 100644 .changeset/thick-donkeys-carry.md diff --git a/.changeset/thick-donkeys-carry.md b/.changeset/thick-donkeys-carry.md new file mode 100644 index 0000000000..41cd319654 --- /dev/null +++ b/.changeset/thick-donkeys-carry.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-scaffolder-backend': patch +--- + +Migrate from the `command-exists-promise` dependency to `command-exists`. diff --git a/plugins/scaffolder-backend/package.json b/plugins/scaffolder-backend/package.json index 8eb6097fb6..f437eff300 100644 --- a/plugins/scaffolder-backend/package.json +++ b/plugins/scaffolder-backend/package.json @@ -41,7 +41,7 @@ "@types/express": "^4.17.6", "@types/git-url-parse": "^9.0.0", "azure-devops-node-api": "^10.1.1", - "command-exists-promise": "^2.0.2", + "command-exists": "^1.2.9", "compression": "^1.7.4", "cors": "^2.8.5", "cross-fetch": "^3.0.6", diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/templater/cookiecutter.test.ts b/plugins/scaffolder-backend/src/scaffolder/stages/templater/cookiecutter.test.ts index 59fb72d515..a9227c742e 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/templater/cookiecutter.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/templater/cookiecutter.test.ts @@ -18,7 +18,7 @@ const runCommand = jest.fn(); const commandExists = jest.fn(); jest.mock('./helpers', () => ({ runCommand })); -jest.mock('command-exists-promise', () => commandExists); +jest.mock('command-exists', () => commandExists); jest.mock('fs-extra'); import { ContainerRunner } from '@backstage/backend-common'; diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/templater/cookiecutter.ts b/plugins/scaffolder-backend/src/scaffolder/stages/templater/cookiecutter.ts index 8819a90d63..541c433a6c 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/templater/cookiecutter.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/templater/cookiecutter.ts @@ -21,7 +21,7 @@ import path from 'path'; import { runCommand } from './helpers'; import { TemplaterBase, TemplaterRunOptions } from './types'; -const commandExists = require('command-exists-promise'); +const commandExists = require('command-exists'); export class CookieCutter implements TemplaterBase { private readonly containerRunner: ContainerRunner; diff --git a/yarn.lock b/yarn.lock index af81214be9..b453ea3693 100644 --- a/yarn.lock +++ b/yarn.lock @@ -9802,11 +9802,6 @@ comma-separated-tokens@^1.0.0: resolved "https://registry.npmjs.org/comma-separated-tokens/-/comma-separated-tokens-1.0.8.tgz#632b80b6117867a158f1080ad498b2fbe7e3f5ea" integrity sha512-GHuDRO12Sypu2cV70d1dkA2EUmXHgntrzbpvOB+Qy+49ypNfGgFQIC2fhhXbnyrJRynDCAARsT7Ou0M6hirpfw== -command-exists-promise@^2.0.2: - version "2.0.2" - resolved "https://registry.npmjs.org/command-exists-promise/-/command-exists-promise-2.0.2.tgz#7beecc4b218299f3c61fa69a4047aa0b36a64a99" - integrity sha512-T6PB6vdFrwnHXg/I0kivM3DqaCGZLjjYSOe0a5WgFKcz1sOnmOeIjnhQPXVXX3QjVbLyTJ85lJkX6lUpukTzaA== - command-exists@^1.2.9: version "1.2.9" resolved "https://registry.npmjs.org/command-exists/-/command-exists-1.2.9.tgz#c50725af3808c8ab0260fd60b01fbfa25b954f69" From ca70bd37d5adcd3b60b910a3f1abf87dde08da40 Mon Sep 17 00:00:00 2001 From: Dominik Henneke Date: Mon, 14 Jun 2021 12:55:19 +0200 Subject: [PATCH 35/93] Move from require to import Signed-off-by: Dominik Henneke --- plugins/scaffolder-backend/package.json | 1 + .../src/scaffolder/stages/templater/cookiecutter.ts | 3 +-- yarn.lock | 5 +++++ 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/plugins/scaffolder-backend/package.json b/plugins/scaffolder-backend/package.json index f437eff300..9c782d7053 100644 --- a/plugins/scaffolder-backend/package.json +++ b/plugins/scaffolder-backend/package.json @@ -66,6 +66,7 @@ "devDependencies": { "@backstage/cli": "^0.7.0", "@backstage/test-utils": "^0.1.13", + "@types/command-exists": "^1.2.0", "@types/fs-extra": "^9.0.1", "@types/mock-fs": "^4.13.0", "@types/supertest": "^2.0.8", diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/templater/cookiecutter.ts b/plugins/scaffolder-backend/src/scaffolder/stages/templater/cookiecutter.ts index 541c433a6c..c0abc44521 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/templater/cookiecutter.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/templater/cookiecutter.ts @@ -16,13 +16,12 @@ import { ContainerRunner } from '@backstage/backend-common'; import { JsonValue } from '@backstage/config'; +import commandExists from 'command-exists'; import fs from 'fs-extra'; import path from 'path'; import { runCommand } from './helpers'; import { TemplaterBase, TemplaterRunOptions } from './types'; -const commandExists = require('command-exists'); - export class CookieCutter implements TemplaterBase { private readonly containerRunner: ContainerRunner; diff --git a/yarn.lock b/yarn.lock index b453ea3693..808c68443e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5508,6 +5508,11 @@ dependencies: "@types/color-convert" "*" +"@types/command-exists@^1.2.0": + version "1.2.0" + resolved "https://registry.npmjs.org/@types/command-exists/-/command-exists-1.2.0.tgz#d97e0ed10097090e4ab0367ed425b0312fad86f3" + integrity sha512-ugsxEJfsCuqMLSuCD4PIJkp5Uk2z6TCMRCgYVuhRo5cYQY3+1xXTQkSlPtkpGHuvWMjS2KTeVQXxkXRACMbM6A== + "@types/compression@^1.7.0": version "1.7.0" resolved "https://registry.npmjs.org/@types/compression/-/compression-1.7.0.tgz#8dc2a56604873cf0dd4e746d9ae4d31ae77b2390" From a6a0ba7ff0d48f4d6f9d20fa78134781526511be Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Mon, 14 Jun 2021 13:38:39 +0200 Subject: [PATCH 36/93] Create flat-dolls-search.md Signed-off-by: Patrik Oldsberg --- .changeset/flat-dolls-search.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/flat-dolls-search.md diff --git a/.changeset/flat-dolls-search.md b/.changeset/flat-dolls-search.md new file mode 100644 index 0000000000..02a7124cb7 --- /dev/null +++ b/.changeset/flat-dolls-search.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-todo-backend': patch +--- + +Bump leasot dependency from 11.5.0 to 12.0.0, removing support for Node.js version 10. From 3108ff7bfd7d1db1d955f2748f3a00b509905481 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Mon, 14 Jun 2021 09:54:46 +0200 Subject: [PATCH 37/93] Make yarn dev for backends respect the PLUGIN_PORT env var MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fredrik Adelöw --- .changeset/fast-trees-arrive.md | 24 +++++++++++++++++++ .changeset/pink-llamas-sniff.md | 12 ++++++++++ .../src/service/standaloneServer.ts.hbs | 7 ++++-- .../src/service/standaloneServer.ts | 4 +++- .../src/service/standaloneServer.ts | 7 ++++-- .../src/service/standaloneServer.ts | 7 ++++-- .../src/service/standaloneServer.ts | 7 ++++-- .../src/service/standaloneServer.ts | 7 ++++-- .../src/service/standaloneServer.ts | 7 ++++-- .../src/service/standaloneServer.ts | 7 ++++-- .../src/service/standaloneServer.ts | 7 ++++-- 11 files changed, 79 insertions(+), 17 deletions(-) create mode 100644 .changeset/fast-trees-arrive.md create mode 100644 .changeset/pink-llamas-sniff.md diff --git a/.changeset/fast-trees-arrive.md b/.changeset/fast-trees-arrive.md new file mode 100644 index 0000000000..f74b6c1c9a --- /dev/null +++ b/.changeset/fast-trees-arrive.md @@ -0,0 +1,24 @@ +--- +'@backstage/cli': patch +--- + +Make `yarn dev` in newly created backend plugins respect the `PLUGIN_PORT` environment variable. + +You can achieve the same in your created backend plugins by making sure to properly call the port and CORS methods on your service builder. Typically in a file named `src/service/standaloneServer.ts` inside your backend plugin package, replace the following: + +```ts +const service = createServiceBuilder(module) + .enableCors({ origin: 'http://localhost:3000' }) + .addRouter('/my-plugin', router); +``` + +With something like the following: + +```ts +let service = createServiceBuilder(module) + .setPort(options.port) + .addRouter('/my-plugin', router); +if (options.enableCors) { + service = service.enableCors({ origin: 'http://localhost:3000' }); +} +``` diff --git a/.changeset/pink-llamas-sniff.md b/.changeset/pink-llamas-sniff.md new file mode 100644 index 0000000000..98addd691c --- /dev/null +++ b/.changeset/pink-llamas-sniff.md @@ -0,0 +1,12 @@ +--- +'@backstage/plugin-app-backend': patch +'@backstage/plugin-badges-backend': patch +'@backstage/plugin-catalog-backend': patch +'@backstage/plugin-code-coverage-backend': patch +'@backstage/plugin-proxy-backend': patch +'@backstage/plugin-rollbar-backend': patch +'@backstage/plugin-search-backend': patch +'@backstage/plugin-techdocs-backend': patch +--- + +Make `yarn dev` respect the `PLUGIN_PORT` environment variable. diff --git a/packages/cli/templates/default-backend-plugin/src/service/standaloneServer.ts.hbs b/packages/cli/templates/default-backend-plugin/src/service/standaloneServer.ts.hbs index 6e38965246..765b6aa0d0 100644 --- a/packages/cli/templates/default-backend-plugin/src/service/standaloneServer.ts.hbs +++ b/packages/cli/templates/default-backend-plugin/src/service/standaloneServer.ts.hbs @@ -34,9 +34,12 @@ export async function startStandaloneServer( logger, }); - const service = createServiceBuilder(module) - .enableCors({ origin: 'http://localhost:3000' }) + let service = createServiceBuilder(module) + .setPort(options.port) .addRouter('/{{id}}', router); + if (options.enableCors) { + service = service.enableCors({ origin: 'http://localhost:3000' }); + } return await service.start().catch(err => { logger.error(err); diff --git a/plugins/app-backend/src/service/standaloneServer.ts b/plugins/app-backend/src/service/standaloneServer.ts index 005d80027b..58267f227d 100644 --- a/plugins/app-backend/src/service/standaloneServer.ts +++ b/plugins/app-backend/src/service/standaloneServer.ts @@ -38,7 +38,9 @@ export async function startStandaloneServer( appPackageName: 'example-app', }); - const service = createServiceBuilder(module).addRouter('', router); + const service = createServiceBuilder(module) + .setPort(options.port) + .addRouter('', router); return await service.start().catch(err => { logger.error(err); diff --git a/plugins/badges-backend/src/service/standaloneServer.ts b/plugins/badges-backend/src/service/standaloneServer.ts index c210efa249..65a78d08e8 100644 --- a/plugins/badges-backend/src/service/standaloneServer.ts +++ b/plugins/badges-backend/src/service/standaloneServer.ts @@ -40,9 +40,12 @@ export async function startStandaloneServer( const router = await createRouter({ config, discovery }); - const service = createServiceBuilder(module) - .enableCors({ origin: 'http://localhost:3000' }) + let service = createServiceBuilder(module) + .setPort(options.port) .addRouter('/badges', router); + if (options.enableCors) { + service = service.enableCors({ origin: 'http://localhost:3000' }); + } return await service.start().catch(err => { logger.error(err); diff --git a/plugins/catalog-backend/src/service/standaloneServer.ts b/plugins/catalog-backend/src/service/standaloneServer.ts index 44a5c0f84d..24a1a9e4cd 100644 --- a/plugins/catalog-backend/src/service/standaloneServer.ts +++ b/plugins/catalog-backend/src/service/standaloneServer.ts @@ -63,9 +63,12 @@ export async function startStandaloneServer( logger, config, }); - const service = createServiceBuilder(module) - .enableCors({ origin: 'http://localhost:3000' }) + let service = createServiceBuilder(module) + .setPort(options.port) .addRouter('/catalog', router); + if (options.enableCors) { + service = service.enableCors({ origin: 'http://localhost:3000' }); + } return await service.start().catch(err => { logger.error(err); process.exit(1); diff --git a/plugins/code-coverage-backend/src/service/standaloneServer.ts b/plugins/code-coverage-backend/src/service/standaloneServer.ts index 2fb9627936..1e9e5131d1 100644 --- a/plugins/code-coverage-backend/src/service/standaloneServer.ts +++ b/plugins/code-coverage-backend/src/service/standaloneServer.ts @@ -61,9 +61,12 @@ export async function startStandaloneServer( logger, }); - const service = createServiceBuilder(module) - .enableCors({ origin: 'http://localhost:3000' }) + let service = createServiceBuilder(module) + .setPort(options.port) .addRouter('/code-coverage', router); + if (options.enableCors) { + service = service.enableCors({ origin: 'http://localhost:3000' }); + } return await service.start().catch(err => { logger.error(err); diff --git a/plugins/proxy-backend/src/service/standaloneServer.ts b/plugins/proxy-backend/src/service/standaloneServer.ts index c64d69e2a4..bd681948bc 100644 --- a/plugins/proxy-backend/src/service/standaloneServer.ts +++ b/plugins/proxy-backend/src/service/standaloneServer.ts @@ -43,9 +43,12 @@ export async function startStandaloneServer( logger, discovery, }); - const service = createServiceBuilder(module) - .enableCors({ origin: 'http://localhost:3000' }) + let service = createServiceBuilder(module) + .setPort(options.port) .addRouter('/proxy', router); + if (options.enableCors) { + service = service.enableCors({ origin: 'http://localhost:3000' }); + } logger.debug('Starting application server...'); diff --git a/plugins/rollbar-backend/src/service/standaloneServer.ts b/plugins/rollbar-backend/src/service/standaloneServer.ts index b30bf6fc6b..aef9741475 100644 --- a/plugins/rollbar-backend/src/service/standaloneServer.ts +++ b/plugins/rollbar-backend/src/service/standaloneServer.ts @@ -38,9 +38,12 @@ export async function startStandaloneServer( const router = await createRouter({ logger, config }); - const service = createServiceBuilder(module) - .enableCors({ origin: 'http://localhost:3000' }) + let service = createServiceBuilder(module) + .setPort(options.port) .addRouter('/catalog', router); + if (options.enableCors) { + service = service.enableCors({ origin: 'http://localhost:3000' }); + } return await service.start().catch(err => { logger.error(err); diff --git a/plugins/search-backend/src/service/standaloneServer.ts b/plugins/search-backend/src/service/standaloneServer.ts index df42e3dda2..19ea40ccf7 100644 --- a/plugins/search-backend/src/service/standaloneServer.ts +++ b/plugins/search-backend/src/service/standaloneServer.ts @@ -44,9 +44,12 @@ export async function startStandaloneServer( logger, }); - const service = createServiceBuilder(module) - .enableCors({ origin: 'http://localhost:3000' }) + let service = createServiceBuilder(module) + .setPort(options.port) .addRouter('/search', router); + if (options.enableCors) { + service = service.enableCors({ origin: 'http://localhost:3000' }); + } return await service.start().catch(err => { logger.error(err); diff --git a/plugins/techdocs-backend/src/service/standaloneServer.ts b/plugins/techdocs-backend/src/service/standaloneServer.ts index af03987870..09379f4296 100644 --- a/plugins/techdocs-backend/src/service/standaloneServer.ts +++ b/plugins/techdocs-backend/src/service/standaloneServer.ts @@ -88,9 +88,12 @@ export async function startStandaloneServer( config, discovery, }); - const service = createServiceBuilder(module) - .enableCors({ origin: 'http://localhost:3000' }) + let service = createServiceBuilder(module) + .setPort(options.port) .addRouter('/techdocs', router); + if (options.enableCors) { + service = service.enableCors({ origin: 'http://localhost:3000' }); + } return await service.start().catch(err => { logger.error(err); process.exit(1); From d4644f5920ff2eedb363d373680d89af35b36359 Mon Sep 17 00:00:00 2001 From: Dominik Henneke Date: Mon, 14 Jun 2021 17:11:09 +0200 Subject: [PATCH 38/93] Use the Backstage `Link` component in the `Button` Signed-off-by: Dominik Henneke --- .changeset/yellow-schools-matter.md | 5 +++++ packages/core/src/components/Button/Button.tsx | 14 ++++++++------ 2 files changed, 13 insertions(+), 6 deletions(-) create mode 100644 .changeset/yellow-schools-matter.md diff --git a/.changeset/yellow-schools-matter.md b/.changeset/yellow-schools-matter.md new file mode 100644 index 0000000000..b159828b88 --- /dev/null +++ b/.changeset/yellow-schools-matter.md @@ -0,0 +1,5 @@ +--- +'@backstage/core': patch +--- + +Use the Backstage `Link` component in the `Button` diff --git a/packages/core/src/components/Button/Button.tsx b/packages/core/src/components/Button/Button.tsx index ca45b3da7f..c678b9a9db 100644 --- a/packages/core/src/components/Button/Button.tsx +++ b/packages/core/src/components/Button/Button.tsx @@ -14,17 +14,19 @@ * limitations under the License. */ -import React, { ComponentProps } from 'react'; -import { Button as MaterialButton } from '@material-ui/core'; -import { Link as RouterLink } from 'react-router-dom'; +import { + Button as MaterialButton, + ButtonProps as MaterialButtonProps, +} from '@material-ui/core'; +import React from 'react'; +import { Link, LinkProps } from '../Link'; -type Props = ComponentProps & - ComponentProps; +type Props = MaterialButtonProps & Omit; /** * Thin wrapper on top of material-ui's Button component * Makes the Button to utilise react-router */ export const Button = React.forwardRef((props, ref) => ( - + )); From 938aee2fbdd79c1a1faca1ef0b78316ccf837505 Mon Sep 17 00:00:00 2001 From: Dominik Henneke Date: Mon, 14 Jun 2021 17:11:25 +0200 Subject: [PATCH 39/93] Fix the link to the documentation page when no owned documents are displayed Signed-off-by: Dominik Henneke --- .changeset/honest-rabbits-divide.md | 5 +++++ plugins/techdocs/src/home/components/DocsTable.tsx | 1 - 2 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 .changeset/honest-rabbits-divide.md diff --git a/.changeset/honest-rabbits-divide.md b/.changeset/honest-rabbits-divide.md new file mode 100644 index 0000000000..bf4b457383 --- /dev/null +++ b/.changeset/honest-rabbits-divide.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-techdocs': patch +--- + +Fix the link to the documentation page when no owned documents are displayed diff --git a/plugins/techdocs/src/home/components/DocsTable.tsx b/plugins/techdocs/src/home/components/DocsTable.tsx index e62435fbe9..e964a9cf00 100644 --- a/plugins/techdocs/src/home/components/DocsTable.tsx +++ b/plugins/techdocs/src/home/components/DocsTable.tsx @@ -111,7 +111,6 @@ export const DocsTable = ({ action={ - )} + All your APIs - +
+ +
+
+ + +
); diff --git a/plugins/api-docs/src/components/ApiExplorerTable/ApiExplorerTable.test.tsx b/plugins/api-docs/src/components/ApiExplorerTable/ApiExplorerTable.test.tsx deleted file mode 100644 index bcb310be46..0000000000 --- a/plugins/api-docs/src/components/ApiExplorerTable/ApiExplorerTable.test.tsx +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { Entity } from '@backstage/catalog-model'; -import { ApiProvider, ApiRegistry } from '@backstage/core'; -import { wrapInTestApp } from '@backstage/test-utils'; -import { render } from '@testing-library/react'; -import * as React from 'react'; -import { apiDocsConfigRef } from '../../config'; -import { ApiExplorerTable } from './ApiExplorerTable'; - -const entities: Entity[] = [ - { - apiVersion: 'backstage.io/v1alpha1', - kind: 'API', - metadata: { name: 'api1' }, - spec: { type: 'openapi' }, - }, - { - apiVersion: 'backstage.io/v1alpha1', - kind: 'API', - metadata: { name: 'api2' }, - spec: { type: 'openapi' }, - }, - { - apiVersion: 'backstage.io/v1alpha1', - kind: 'API', - metadata: { name: 'api3' }, - spec: { type: 'grpc' }, - }, -]; - -const apiRegistry = ApiRegistry.with(apiDocsConfigRef, { - getApiDefinitionWidget: () => undefined, -}); - -describe('ApiCatalogTable component', () => { - it('should render error message when error is passed in props', async () => { - const rendered = render( - wrapInTestApp( - - - , - ), - ); - const errorMessage = await rendered.findByText( - /Could not fetch catalog entities./, - ); - expect(errorMessage).toBeInTheDocument(); - }); - - it('should display entity names when loading has finished and no error occurred', async () => { - const rendered = render( - wrapInTestApp( - - - , - ), - ); - expect(rendered.getByText(/api1/)).toBeInTheDocument(); - expect(rendered.getByText(/api2/)).toBeInTheDocument(); - expect(rendered.getByText(/api3/)).toBeInTheDocument(); - }); -}); diff --git a/plugins/api-docs/src/components/ApiExplorerTable/ApiExplorerTable.tsx b/plugins/api-docs/src/components/ApiExplorerTable/ApiExplorerTable.tsx deleted file mode 100644 index 78b5f1d240..0000000000 --- a/plugins/api-docs/src/components/ApiExplorerTable/ApiExplorerTable.tsx +++ /dev/null @@ -1,216 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { - ApiEntityV1alpha1, - Entity, - EntityName, - RELATION_OWNED_BY, - RELATION_PART_OF, -} from '@backstage/catalog-model'; -import { - CodeSnippet, - OverflowTooltip, - Table, - TableColumn, - TableFilter, - TableState, - useQueryParamState, - WarningPanel, -} from '@backstage/core'; -import { - EntityRefLink, - EntityRefLinks, - formatEntityRefTitle, - getEntityRelations, -} from '@backstage/plugin-catalog-react'; -import { Chip } from '@material-ui/core'; -import React from 'react'; -import { ApiTypeTitle } from '../ApiDefinitionCard'; - -type EntityRow = { - entity: ApiEntityV1alpha1; - resolved: { - name: string; - partOfSystemRelationTitle?: string; - partOfSystemRelations: EntityName[]; - ownedByRelationsTitle?: string; - ownedByRelations: EntityName[]; - }; -}; - -const columns: TableColumn[] = [ - { - title: 'Name', - field: 'resolved.name', - highlight: true, - render: ({ entity }) => ( - - ), - }, - { - title: 'System', - field: 'resolved.partOfSystemRelationTitle', - render: ({ resolved }) => ( - - ), - }, - { - title: 'Owner', - field: 'resolved.ownedByRelationsTitle', - render: ({ resolved }) => ( - - ), - }, - { - title: 'Lifecycle', - field: 'entity.spec.lifecycle', - }, - { - title: 'Type', - field: 'entity.spec.type', - render: ({ entity }) => , - }, - { - title: 'Description', - field: 'entity.metadata.description', - render: ({ entity }) => ( - - ), - width: 'auto', - }, - { - title: 'Tags', - field: 'entity.metadata.tags', - cellStyle: { - padding: '0px 16px 0px 20px', - }, - render: ({ entity }) => ( - <> - {entity.metadata.tags && - entity.metadata.tags.map(t => ( - - ))} - - ), - }, -]; - -const filters: TableFilter[] = [ - { - column: 'Owner', - type: 'select', - }, - { - column: 'Type', - type: 'multiple-select', - }, - { - column: 'Lifecycle', - type: 'multiple-select', - }, - { - column: 'Tags', - type: 'checkbox-tree', - }, -]; - -type ExplorerTableProps = { - entities: Entity[]; - loading: boolean; - error?: any; -}; - -export const ApiExplorerTable = ({ - entities, - loading, - error, -}: ExplorerTableProps) => { - const [queryParamState, setQueryParamState] = useQueryParamState( - 'apiTable', - ); - - if (error) { - return ( - - - - ); - } - - const rows = entities.map(entity => { - const partOfSystemRelations = getEntityRelations(entity, RELATION_PART_OF, { - kind: 'system', - }); - const ownedByRelations = getEntityRelations(entity, RELATION_OWNED_BY); - - return { - entity: entity as ApiEntityV1alpha1, - resolved: { - name: formatEntityRefTitle(entity, { - defaultKind: 'API', - }), - ownedByRelationsTitle: ownedByRelations - .map(r => formatEntityRefTitle(r, { defaultKind: 'group' })) - .join(', '), - ownedByRelations, - partOfSystemRelationTitle: partOfSystemRelations - .map(r => - formatEntityRefTitle(r, { - defaultKind: 'system', - }), - ) - .join(', '), - partOfSystemRelations, - }, - }; - }); - - return ( - - isLoading={loading} - columns={columns} - options={{ - paging: true, - pageSize: 20, - pageSizeOptions: [20, 50, 100], - actionsColumnIndex: -1, - loadingType: 'linear', - padding: 'dense', - showEmptyDataSourceMessage: !loading, - }} - data={rows} - filters={filters} - initialState={queryParamState} - onStateChange={setQueryParamState} - /> - ); -}; diff --git a/plugins/api-docs/src/components/ApiExplorerTable/index.ts b/plugins/api-docs/src/components/ApiExplorerTable/index.ts deleted file mode 100644 index a9c79861e8..0000000000 --- a/plugins/api-docs/src/components/ApiExplorerTable/index.ts +++ /dev/null @@ -1,17 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -export { ApiExplorerTable } from './ApiExplorerTable'; diff --git a/plugins/catalog/src/components/CatalogTable/index.ts b/plugins/catalog/src/components/CatalogTable/index.ts index 280d5b4bcb..460720245e 100644 --- a/plugins/catalog/src/components/CatalogTable/index.ts +++ b/plugins/catalog/src/components/CatalogTable/index.ts @@ -15,3 +15,4 @@ */ export { CatalogTable } from './CatalogTable'; +export type { EntityRow } from './types'; diff --git a/plugins/catalog/src/components/CreateComponentButton/CreateComponentButton.tsx b/plugins/catalog/src/components/CreateComponentButton/CreateComponentButton.tsx index 896e9bada3..8c30c9f9be 100644 --- a/plugins/catalog/src/components/CreateComponentButton/CreateComponentButton.tsx +++ b/plugins/catalog/src/components/CreateComponentButton/CreateComponentButton.tsx @@ -20,7 +20,12 @@ import { Button } from '@material-ui/core'; import { useRouteRef } from '@backstage/core'; import { createComponentRouteRef } from '../../routes'; -export const CreateComponentButton = () => { +type CreateComponentButtonProps = { + buttonLabel?: string; +}; +export const CreateComponentButton = ({ + buttonLabel, +}: CreateComponentButtonProps) => { const createComponentLink = useRouteRef(createComponentRouteRef); if (!createComponentLink) return null; @@ -32,7 +37,7 @@ export const CreateComponentButton = () => { color="primary" to={createComponentLink()} > - Create Component + {buttonLabel ?? 'Create Component'} ); }; diff --git a/plugins/catalog/src/index.ts b/plugins/catalog/src/index.ts index 1596de0e8d..aca1db8940 100644 --- a/plugins/catalog/src/index.ts +++ b/plugins/catalog/src/index.ts @@ -18,6 +18,7 @@ export * from './components/AboutCard'; export { CatalogLayout } from './components/CatalogPage'; export { CatalogResultListItem } from './components/CatalogResultListItem'; export { CatalogTable } from './components/CatalogTable'; +export type { EntityRow } from './components/CatalogTable'; export { CreateComponentButton } from './components/CreateComponentButton'; export { EntityLayout } from './components/EntityLayout'; export * from './components/EntityOrphanWarning'; From 172c973247f64c62e21298e5ca58d5c55f9ca7b8 Mon Sep 17 00:00:00 2001 From: Phil Kuang Date: Thu, 10 Jun 2021 13:00:43 -0400 Subject: [PATCH 62/93] feat(pickers): implement EntityLifecyclePicker and EntityOwnerPicker Signed-off-by: Phil Kuang --- .changeset/chilly-ants-taste.md | 5 + .../ApiExplorerPage/ApiExplorerPage.tsx | 4 + .../EntityLifecyclePicker.test.tsx | 139 ++++++++++++++++++ .../EntityLifecyclePicker.tsx | 85 +++++++++++ .../components/EntityLifecyclePicker/index.ts | 17 +++ .../EntityOwnerPicker.test.tsx | 139 ++++++++++++++++++ .../EntityOwnerPicker/EntityOwnerPicker.tsx | 83 +++++++++++ .../src/components/EntityOwnerPicker/index.ts | 17 +++ plugins/catalog-react/src/components/index.ts | 2 + .../src/hooks/useEntityListProvider.tsx | 4 + plugins/catalog-react/src/types.ts | 16 ++ 11 files changed, 511 insertions(+) create mode 100644 .changeset/chilly-ants-taste.md create mode 100644 plugins/catalog-react/src/components/EntityLifecyclePicker/EntityLifecyclePicker.test.tsx create mode 100644 plugins/catalog-react/src/components/EntityLifecyclePicker/EntityLifecyclePicker.tsx create mode 100644 plugins/catalog-react/src/components/EntityLifecyclePicker/index.ts create mode 100644 plugins/catalog-react/src/components/EntityOwnerPicker/EntityOwnerPicker.test.tsx create mode 100644 plugins/catalog-react/src/components/EntityOwnerPicker/EntityOwnerPicker.tsx create mode 100644 plugins/catalog-react/src/components/EntityOwnerPicker/index.ts diff --git a/.changeset/chilly-ants-taste.md b/.changeset/chilly-ants-taste.md new file mode 100644 index 0000000000..2c68e374f8 --- /dev/null +++ b/.changeset/chilly-ants-taste.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-react': patch +--- + +Implement a `EntityLifecyclePicker` and `EntityOwnerPicker` diff --git a/plugins/api-docs/src/components/ApiExplorerPage/ApiExplorerPage.tsx b/plugins/api-docs/src/components/ApiExplorerPage/ApiExplorerPage.tsx index 36cd988c8c..e5dffa7e73 100644 --- a/plugins/api-docs/src/components/ApiExplorerPage/ApiExplorerPage.tsx +++ b/plugins/api-docs/src/components/ApiExplorerPage/ApiExplorerPage.tsx @@ -22,7 +22,9 @@ import { } from '@backstage/core'; import { EntityKindPicker, + EntityLifecyclePicker, EntityListProvider, + EntityOwnerPicker, EntityTagPicker, EntityTypePicker, UserListFilterKind, @@ -71,6 +73,8 @@ export const ApiExplorerPage = ({