From a35b6459d50b3979509365c532a51932f7c53ca9 Mon Sep 17 00:00:00 2001 From: Marley Powell Date: Thu, 21 Oct 2021 21:53:51 +0100 Subject: [PATCH 001/505] feat: Created `AzurePullRequestsIcon` component. Signed-off-by: Marley Powell --- .../AzurePullRequestsIcon.tsx | 28 +++++++++++++++++++ .../components/AzurePullRequestsIcon/index.ts | 17 +++++++++++ 2 files changed, 45 insertions(+) create mode 100644 plugins/azure-devops/src/components/AzurePullRequestsIcon/AzurePullRequestsIcon.tsx create mode 100644 plugins/azure-devops/src/components/AzurePullRequestsIcon/index.ts diff --git a/plugins/azure-devops/src/components/AzurePullRequestsIcon/AzurePullRequestsIcon.tsx b/plugins/azure-devops/src/components/AzurePullRequestsIcon/AzurePullRequestsIcon.tsx new file mode 100644 index 0000000000..3159556788 --- /dev/null +++ b/plugins/azure-devops/src/components/AzurePullRequestsIcon/AzurePullRequestsIcon.tsx @@ -0,0 +1,28 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { SvgIcon, SvgIconProps } from '@material-ui/core'; + +import React from 'react'; + +export const AzurePullRequestsIcon = (props: SvgIconProps) => ( + + + +); diff --git a/plugins/azure-devops/src/components/AzurePullRequestsIcon/index.ts b/plugins/azure-devops/src/components/AzurePullRequestsIcon/index.ts new file mode 100644 index 0000000000..1f49119b4b --- /dev/null +++ b/plugins/azure-devops/src/components/AzurePullRequestsIcon/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { AzurePullRequestsIcon } from './AzurePullRequestsIcon'; From 7ad9a07b2789e27792b0bb14b0dbf7b6d4ad1554 Mon Sep 17 00:00:00 2001 From: Greg Bomkamp Date: Thu, 28 Oct 2021 10:43:22 -0400 Subject: [PATCH 002/505] Adding config prop `usePluginSchemas` to allow plugins using the `pg` client to create their own management schemas in the db. This allows `pg` client plugins to work in separate schemas in the same db. Signed-off-by: Greg Bomkamp --- .changeset/pretty-moons-drive.md | 5 + packages/backend-common/config.d.ts | 10 + .../src/database/DatabaseManager.test.ts | 193 +++++++++++++++++- .../src/database/DatabaseManager.ts | 87 ++++++-- .../src/database/connection.test.ts | 75 +++++++ .../backend-common/src/database/connection.ts | 34 +++ .../connectors/defaultSchemaOverride.test.ts | 25 +++ .../connectors/defaultSchemaOverride.ts | 29 +++ .../src/database/connectors/postgres.test.ts | 18 ++ .../src/database/connectors/postgres.ts | 26 +++ packages/backend-common/src/database/types.ts | 17 ++ 11 files changed, 497 insertions(+), 22 deletions(-) create mode 100644 .changeset/pretty-moons-drive.md create mode 100644 packages/backend-common/src/database/connectors/defaultSchemaOverride.test.ts create mode 100644 packages/backend-common/src/database/connectors/defaultSchemaOverride.ts diff --git a/.changeset/pretty-moons-drive.md b/.changeset/pretty-moons-drive.md new file mode 100644 index 0000000000..3c3b7968c6 --- /dev/null +++ b/.changeset/pretty-moons-drive.md @@ -0,0 +1,5 @@ +--- +'@backstage/backend-common': patch +--- + +Adding config prop `usePluginSchemas` to allow plugins using the `pg` client to create their own management schemas in the db. This allows `pg` client plugins to work in separate schemas in the same db. diff --git a/packages/backend-common/config.d.ts b/packages/backend-common/config.d.ts index bc5e026375..4476a946cf 100644 --- a/packages/backend-common/config.d.ts +++ b/packages/backend-common/config.d.ts @@ -69,6 +69,16 @@ export interface Config { * Defaults to true if unspecified. */ ensureExists?: boolean; + /** + * Whether plugins should use their own schemas instead of databases. If enabled, + * each plugin will create a schema in the configured database instance + * using the `pluginId` as its schema name. + * + * NOTE: Currently only supported by the `pg` client. + * + * @default false + */ + usePluginSchemas?: boolean; /** Plugin specific database configuration and client override */ plugin?: { [pluginId: string]: { diff --git a/packages/backend-common/src/database/DatabaseManager.test.ts b/packages/backend-common/src/database/DatabaseManager.test.ts index e839123908..e639f14ee8 100644 --- a/packages/backend-common/src/database/DatabaseManager.test.ts +++ b/packages/backend-common/src/database/DatabaseManager.test.ts @@ -15,13 +15,18 @@ */ import { ConfigReader } from '@backstage/config'; import { omit } from 'lodash'; -import { createDatabaseClient, ensureDatabaseExists } from './connection'; +import { + createDatabaseClient, + ensureDatabaseExists, + ensureSchemaExists, +} from './connection'; import { DatabaseManager } from './DatabaseManager'; jest.mock('./connection', () => ({ ...jest.requireActual('./connection'), createDatabaseClient: jest.fn(), ensureDatabaseExists: jest.fn(), + ensureSchemaExists: jest.fn(), })); describe('DatabaseManager', () => { @@ -314,5 +319,191 @@ describe('DatabaseManager', () => { expect.stringContaining('userdbname'), ); }); + + it('plugin sets schema override for pg client', async () => { + const overrideConfig = { + backend: { + database: { + client: 'pg', + usePluginSchemas: true, + connection: { + host: 'localhost', + user: 'foo', + password: 'bar', + database: 'foodb', + }, + }, + }, + }; + const testManager = DatabaseManager.fromConfig( + new ConfigReader(overrideConfig), + ); + const pluginId = 'schemaoverride'; + await testManager.forPlugin(pluginId).getClient(); + + const mockCalls = mocked(createDatabaseClient).mock.calls.splice(-1); + const [baseConfig, overrides] = mockCalls[0]; + + expect(baseConfig.get()).toMatchObject({ + client: 'pg', + connection: config.backend.database.connection, + }); + + expect(overrides).toMatchObject({ + searchPath: [pluginId], + }); + }); + + it('plugin does not provide schema override for non pg client', async () => { + const testManager = DatabaseManager.fromConfig( + new ConfigReader({ + backend: { + database: { + client: 'sqlite3', + usePluginSchemas: true, + connection: { + host: 'localhost', + user: 'foo', + password: 'bar', + database: 'foodb', + }, + }, + }, + }), + ); + const pluginId = 'any-plugin'; + await testManager.forPlugin(pluginId).getClient(); + + const mockCalls = mocked(createDatabaseClient).mock.calls.splice(-1); + const [baseConfig, overrides] = mockCalls[0]; + + expect(baseConfig.get()).toMatchObject({ + client: 'sqlite3', + connection: config.backend.database.connection, + }); + + expect(overrides).not.toHaveProperty('searchPath'); + }); + + it('plugin does not provide schema override if usePluginSchemas is not provided', async () => { + const testManager = DatabaseManager.fromConfig( + new ConfigReader({ + backend: { + database: { + client: 'pg', + usePluginSchemas: false, + connection: 'some-file-path', + }, + }, + }), + ); + + const pluginId = 'any-plugin'; + await testManager.forPlugin(pluginId).getClient(); + + const mockCalls = mocked(createDatabaseClient).mock.calls.splice(-1); + const [_baseConfig, overrides] = mockCalls[0]; + + expect(overrides).not.toHaveProperty('searchPath'); + }); + + it('plugin does not provide schema override if usePluginSchemas is false', async () => { + const testManager = DatabaseManager.fromConfig( + new ConfigReader({ + backend: { + database: { + client: 'pg', + connection: { + host: 'localhost', + user: 'foo', + password: 'bar', + database: 'foodb', + }, + }, + }, + }), + ); + + const pluginId = 'schemaoverride'; + await testManager.forPlugin(pluginId).getClient(); + + const mockCalls = mocked(createDatabaseClient).mock.calls.splice(-1); + const [_baseConfig, overrides] = mockCalls[0]; + + expect(overrides).not.toHaveProperty('searchPath'); + }); + + it('usePluginSchemas ensures that each plugin schema exists', async () => { + const testManager = DatabaseManager.fromConfig( + new ConfigReader({ + backend: { + database: { + client: 'pg', + usePluginSchemas: true, + connection: { + host: 'localhost', + user: 'foo', + password: 'bar', + database: 'foodb', + }, + }, + }, + }), + ); + const pluginId = 'testdbname'; + await testManager.forPlugin(pluginId).getClient(); + + const mockCalls = mocked(ensureSchemaExists).mock.calls; + const [_, schemaName] = mockCalls[0]; + + expect(schemaName).toEqual('testdbname'); + }); + + it('usePluginSchemas allows connection overrides for plugins', async () => { + const testManager = DatabaseManager.fromConfig( + new ConfigReader({ + backend: { + database: { + client: 'pg', + usePluginSchemas: true, + connection: { + host: 'localhost', + user: 'foo', + password: 'bar', + database: 'foodb', + }, + plugin: { + testdbname: { + connection: { + database: 'database_name_overriden', + host: 'newhost', + }, + }, + }, + }, + }, + }), + ); + const pluginId = 'testdbname'; + await testManager.forPlugin(pluginId).getClient(); + + const mockCalls = mocked(createDatabaseClient).mock.calls.splice(-1); + const [baseConfig, overrides] = mockCalls[0]; + + expect(baseConfig.get()).toMatchObject({ + client: 'pg', + connection: { + database: 'database_name_overriden', + host: 'newhost', + user: 'foo', + password: 'bar', + }, + }); + expect(overrides).toHaveProperty('searchPath', ['testdbname']); + expect(overrides).toHaveProperty( + 'connection.database', + 'database_name_overriden', + ); + }); }); }); diff --git a/packages/backend-common/src/database/DatabaseManager.ts b/packages/backend-common/src/database/DatabaseManager.ts index c0511c6b78..efd779e4ed 100644 --- a/packages/backend-common/src/database/DatabaseManager.ts +++ b/packages/backend-common/src/database/DatabaseManager.ts @@ -18,12 +18,15 @@ import { omit } from 'lodash'; import { Config, ConfigReader } from '@backstage/config'; import { JsonObject } from '@backstage/types'; import { - createDatabaseClient, ensureDatabaseExists, createNameOverride, normalizeConnection, + createSchemaOverride, + ensureSchemaExists, + createDatabaseClient, } from './connection'; import { PluginDatabaseManager } from './types'; +import { mergeDatabaseConfig } from './config'; /** * Provides a config lookup path for a plugin's config block. @@ -79,14 +82,15 @@ export class DatabaseManager { * Provides the canonical database name for a given plugin. * * This method provides the effective database name which is determined using global - * and plugin specific database config. If no explicit database name is configured, - * this method will provide a generated name which is the pluginId prefixed with - * 'backstage_plugin_'. + * and plugin specific database config. If no explicit database name is configured + * and usePluginSchemas is not enabled, this method will provide a generated name + * which is the pluginId prefixed with 'backstage_plugin_'. If `usePluginSchemas` is + * enabled, it will fallback to using the default database for the knex instance. * * @param pluginId Lookup the database name for given plugin * @returns String representing the plugin's database name */ - private getDatabaseName(pluginId: string): string { + private getDatabaseName(pluginId: string): string | undefined { const connection = this.getConnectionConfig(pluginId); if (this.getClientType(pluginId).client === 'sqlite3') { @@ -95,11 +99,16 @@ export class DatabaseManager { (connection as Knex.Sqlite3ConnectionConfig)?.filename ?? ':memory:' ); } + + const databaseName = (connection as Knex.ConnectionConfig)?.database; + + // usePluginSchemas enabled should use overridden databaseName if supplied or fallback to default knex database + if (this.getUsePluginSchemasConfig()) { + return databaseName; + } + // all other supported databases should fallback to an auto-prefixed name - return ( - (connection as Knex.ConnectionConfig)?.database ?? - `${this.prefix}${pluginId}` - ); + return databaseName ?? `${this.prefix}${pluginId}`; } /** @@ -136,13 +145,18 @@ export class DatabaseManager { ); } + private getUsePluginSchemasConfig(): boolean { + return this.config.getOptionalBoolean('usePluginSchemas') ?? false; + } + /** * Provides a Knex connection plugin config by combining base and plugin config. * * This method provides a baseConfig for a plugin database connector. If the client type * has not been overridden, the global connection config will be included with plugin * specific config as the base. Values from the plugin connection take precedence over the - * base. Base database name is omitted for all supported databases excluding SQLite. + * base. Base database name is omitted for all supported databases excluding SQLite unless + * `usePluginSchemas` is enabled. */ private getConnectionConfig( pluginId: string, @@ -153,10 +167,13 @@ export class DatabaseManager { this.config.get('connection'), this.config.getString('client'), ); - // As databases cannot be shared, the `database` property from the base connection - // is omitted. SQLite3's `filename` property is an exception as this is used as a + // Databases cannot be shared unless the usePluginSchemas is enabled. The + // `database` property from the base connection is omitted unless usePluginSchemas + // is enabled. SQLite3's `filename` property is an exception as this is used as a // directory elsewhere so we preserve `filename`. - baseConnection = omit(baseConnection, 'database'); + if (!this.getUsePluginSchemasConfig()) { + baseConnection = omit(baseConnection, 'database'); + } // get and normalize optional plugin specific database connection const connection = normalizeConnection( @@ -187,6 +204,16 @@ export class DatabaseManager { }; } + /** + * Provides a partial Knex.Config database schema override for a given plugin. + * + * @param pluginId Target plugin to get database schema override + * @returns Partial Knex.Config with database schema override + */ + private getSchemaOverrides(pluginId: string): Knex.Config | undefined { + return createSchemaOverride(this.getClientType(pluginId).client, pluginId); + } + /** * Provides a partial Knex.Config database name override for a given plugin. * @@ -194,10 +221,13 @@ export class DatabaseManager { * @returns Partial Knex.Config with database name override */ private getDatabaseOverrides(pluginId: string): Knex.Config { - return createNameOverride( - this.getClientType(pluginId).client, - this.getDatabaseName(pluginId), - ); + const databaseNameOverride = this.getDatabaseName(pluginId); + return databaseNameOverride + ? createNameOverride( + this.getClientType(pluginId).client, + databaseNameOverride, + ) + : {}; } /** @@ -211,8 +241,8 @@ export class DatabaseManager { this.getConfigForPlugin(pluginId) as JsonObject, ); - if (this.getEnsureExistsConfig(pluginId)) { - const databaseName = this.getDatabaseName(pluginId); + const databaseName = this.getDatabaseName(pluginId); + if (databaseName && this.getEnsureExistsConfig(pluginId)) { try { await ensureDatabaseExists(pluginConfig, databaseName); } catch (error) { @@ -222,9 +252,24 @@ export class DatabaseManager { } } - return createDatabaseClient( - pluginConfig, + let schemaOverrides; + if (this.getUsePluginSchemasConfig()) { + try { + schemaOverrides = this.getSchemaOverrides(pluginId); + await ensureSchemaExists(pluginConfig, pluginId); + } catch (error) { + throw new Error( + `Failed to connect to the database to make sure that schema for plugin '${pluginId}' exists, ${error}`, + ); + } + } + + const databaseClientOverrides = mergeDatabaseConfig( + {}, this.getDatabaseOverrides(pluginId), + schemaOverrides, ); + + return createDatabaseClient(pluginConfig, databaseClientOverrides); } } diff --git a/packages/backend-common/src/database/connection.test.ts b/packages/backend-common/src/database/connection.test.ts index 869722ddf7..6f02e7b2ed 100644 --- a/packages/backend-common/src/database/connection.test.ts +++ b/packages/backend-common/src/database/connection.test.ts @@ -18,8 +18,24 @@ import { ConfigReader } from '@backstage/config'; import { createDatabaseClient, createNameOverride, + createSchemaOverride, + ensureSchemaExists, parseConnectionString, } from './connection'; +import { pgConnector } from './connectors'; + +const mocked = (f: Function) => f as jest.Mock; + +jest.mock('./connectors', () => { + const connectors = jest.requireActual('./connectors'); + return { + ...connectors, + pgConnector: { + ...connectors.pgConnector, + ensureSchemaExists: jest.fn(), + }, + }; +}); describe('database connection', () => { describe('createDatabaseClient', () => { @@ -152,4 +168,63 @@ describe('database connection', () => { expect(() => parseConnectionString('sqlite://')).toThrow(); }); }); + + describe('createSchemaOverride', () => { + it('returns Knex config for postgres', () => { + expect(createSchemaOverride('pg', 'testpg')).toHaveProperty( + 'searchPath', + ['testpg'], + ); + }); + + it('throws error for sqlite', () => { + expect(createSchemaOverride('sqlite3', 'testsqlite')).toBeUndefined(); + }); + + it('returns Knex config for mysql', () => { + expect(createSchemaOverride('mysql', 'testmysql')).toBeUndefined(); + }); + + it('throws an error for unknown connection', () => { + expect(createSchemaOverride('unknown', 'testname')).toBeUndefined(); + }); + }); + + describe('ensureSchemaExists', () => { + it('returns sucessfully with pg client', async () => { + await ensureSchemaExists( + new ConfigReader({ + client: 'pg', + schema: 'catalog', + connection: 'postgresql://testuser:testpass@acme:5432/userdbname', + }), + 'catalog', + ); + + const mockCalls = mocked( + pgConnector.ensureSchemaExists as Function, + ).mock.calls.splice(-1); + const [baseConfig, schemaName] = mockCalls[0]; + + expect(baseConfig.get()).toMatchObject({ + client: 'pg', + connection: 'postgresql://testuser:testpass@acme:5432/userdbname', + }); + + expect(schemaName).toEqual('catalog'); + }); + + it('throws error for non pg client', () => { + return expect( + ensureSchemaExists( + new ConfigReader({ + client: 'sqlite3', + schema: 'catalog', + connection: ':memory:', + }), + 'catalog', + ), + ).resolves.toBeUndefined(); + }); + }); }); diff --git a/packages/backend-common/src/database/connection.ts b/packages/backend-common/src/database/connection.ts index 1c5cce3718..3308a26573 100644 --- a/packages/backend-common/src/database/connection.ts +++ b/packages/backend-common/src/database/connection.ts @@ -82,6 +82,23 @@ export async function ensureDatabaseExists( ); } +/** + * Ensures that the given schemas all exist, creating them if they do not. + * + * @public + */ +export async function ensureSchemaExists( + dbConfig: Config, + ...schemas: Array +): Promise { + const client: DatabaseClient = dbConfig.getString('client'); + + return await ConnectorMapping[client]?.ensureSchemaExists?.( + dbConfig, + ...schemas, + ); +} + /** * Provides a Knex.Config object with the provided database name for a given client. */ @@ -99,6 +116,23 @@ export function createNameOverride( } } +/** + * Provides a Knex.Config object with the provided database schema for a given client. Currently only supported by `pg`. + */ +export function createSchemaOverride( + client: string, + name: string, +): Partial { + try { + return ConnectorMapping[client]?.createSchemaOverride?.(name); + } catch (e) { + throw new InputError( + `Unable to create database schema override for '${client}' connector`, + e, + ); + } +} + /** * Parses a connection string for a given client and provides a connection config. */ diff --git a/packages/backend-common/src/database/connectors/defaultSchemaOverride.test.ts b/packages/backend-common/src/database/connectors/defaultSchemaOverride.test.ts new file mode 100644 index 0000000000..2db5ecd454 --- /dev/null +++ b/packages/backend-common/src/database/connectors/defaultSchemaOverride.test.ts @@ -0,0 +1,25 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import defaultSchemaOverride from './defaultSchemaOverride'; + +describe('defaultNameOverride()', () => { + it('returns a partial knex static connection config with searchPath set to [schemaName]', () => { + const schemaName = 'schemaName'; + expect(defaultSchemaOverride(schemaName)).toHaveProperty('searchPath', [ + schemaName, + ]); + }); +}); diff --git a/packages/backend-common/src/database/connectors/defaultSchemaOverride.ts b/packages/backend-common/src/database/connectors/defaultSchemaOverride.ts new file mode 100644 index 0000000000..62ad8ab8c8 --- /dev/null +++ b/packages/backend-common/src/database/connectors/defaultSchemaOverride.ts @@ -0,0 +1,29 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Knex } from 'knex'; + +/** + * Provides a partial knex config with schema name override. + * + * @param name schema name to get config override for + */ +export default function defaultSchemaOverride( + name: string, +): Partial { + return { + searchPath: [name], + }; +} diff --git a/packages/backend-common/src/database/connectors/postgres.test.ts b/packages/backend-common/src/database/connectors/postgres.test.ts index ac988a2880..553d019573 100644 --- a/packages/backend-common/src/database/connectors/postgres.test.ts +++ b/packages/backend-common/src/database/connectors/postgres.test.ts @@ -76,6 +76,24 @@ describe('postgres', () => { }); }); + it('overrides the schema name', () => { + const mockConnection = { + ...createMockConnection(), + schema: 'schemaName', + }; + + expect( + buildPgDatabaseConfig(createConfig(mockConnection), { + searchPath: ['schemaName'], + }), + ).toEqual({ + client: 'pg', + connection: mockConnection, + searchPath: ['schemaName'], + useNullAsDefault: true, + }); + }); + it('adds additional config settings', () => { const mockConnection = createMockConnection(); diff --git a/packages/backend-common/src/database/connectors/postgres.ts b/packages/backend-common/src/database/connectors/postgres.ts index 62de2e001e..823d50f73f 100644 --- a/packages/backend-common/src/database/connectors/postgres.ts +++ b/packages/backend-common/src/database/connectors/postgres.ts @@ -21,6 +21,7 @@ import { ForwardedError } from '@backstage/errors'; import { mergeDatabaseConfig } from '../config'; import { DatabaseConnector } from '../types'; import defaultNameOverride from './defaultNameOverride'; +import defaultSchemaOverride from './defaultSchemaOverride'; /** * Creates a knex postgres database connection @@ -135,6 +136,29 @@ export async function ensurePgDatabaseExists( } } +/** + * Creates the missing Postgres schema if it does not exist + * + * @param dbConfig The database config + * @param schemas The name of the schemas to create + */ +export async function ensurePgSchemaExists( + dbConfig: Config, + ...schemas: Array +): Promise { + const admin = createPgDatabaseClient(dbConfig); + + try { + const ensureSchema = async (database: string) => { + await admin.raw(`CREATE SCHEMA IF NOT EXISTS ??`, [database]); + }; + + await Promise.all(schemas.map(ensureSchema)); + } finally { + await admin.destroy(); + } +} + /** * PostgreSQL database connector. * @@ -143,6 +167,8 @@ export async function ensurePgDatabaseExists( export const pgConnector: DatabaseConnector = Object.freeze({ createClient: createPgDatabaseClient, ensureDatabaseExists: ensurePgDatabaseExists, + ensureSchemaExists: ensurePgSchemaExists, createNameOverride: defaultNameOverride, + createSchemaOverride: defaultSchemaOverride, parseConnectionString: parsePgConnectionString, }); diff --git a/packages/backend-common/src/database/types.ts b/packages/backend-common/src/database/types.ts index edcba1e643..e96f86980b 100644 --- a/packages/backend-common/src/database/types.ts +++ b/packages/backend-common/src/database/types.ts @@ -45,6 +45,11 @@ export interface DatabaseConnector { * database name. */ createNameOverride(name: string): Partial; + /** + * createSchemaOverride provides a partial knex config sufficient to override a + * PostgreSQL schema name within utilizing the `searchPath` knex configuration. + */ + createSchemaOverride?(name: string): Partial; /** * parseConnectionString produces a knex connection config object representing * a database connection string. @@ -64,4 +69,16 @@ export interface DatabaseConnector { dbConfig: Config, ...databases: Array ): Promise; + + /** + * ensureSchemaExists performs a side-effect to ensure schema names passed in are + * present. + * + * Calling this function on schemas which already exist should do nothing. + * Missing schemas should be created if needed. + */ + ensureSchemaExists?( + dbConfig: Config, + ...schemas: Array + ): Promise; } From b8b67b574011074720b9f7d252b9ee2005b9a97b Mon Sep 17 00:00:00 2001 From: Gabriele Mambrini Date: Fri, 5 Nov 2021 09:52:50 +0100 Subject: [PATCH 003/505] Add allowed paths to backend.reading.allow Signed-off-by: Gabriele Mambrini --- .../src/reading/FetchUrlReader.test.ts | 7 +++++++ .../backend-common/src/reading/FetchUrlReader.ts | 15 +++++++++++++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/packages/backend-common/src/reading/FetchUrlReader.test.ts b/packages/backend-common/src/reading/FetchUrlReader.test.ts index 124abb799c..9d6edee765 100644 --- a/packages/backend-common/src/reading/FetchUrlReader.test.ts +++ b/packages/backend-common/src/reading/FetchUrlReader.test.ts @@ -77,6 +77,10 @@ describe('FetchUrlReader', () => { { host: 'example.com:700' }, { host: '*.examples.org' }, { host: '*.examples.org:700' }, + { + host: 'foobar.org', + paths: ['/dir1/'], + }, ], }, }, @@ -106,6 +110,9 @@ describe('FetchUrlReader', () => { expect(predicate(new URL('https://examples.org:700/test'))).toBe(false); expect(predicate(new URL('https://a.examples.org:700/test'))).toBe(true); expect(predicate(new URL('https://a.b.examples.org:700/test'))).toBe(true); + expect(predicate(new URL('https://foobar.org/dir1/subpath'))).toBe(true); + expect(predicate(new URL('https://foobar.org/dir12'))).toBe(false); + expect(predicate(new URL('https://foobar.org/'))).toBe(false); }); describe('read', () => { diff --git a/packages/backend-common/src/reading/FetchUrlReader.ts b/packages/backend-common/src/reading/FetchUrlReader.ts index 732d3b9f59..1ae9677a85 100644 --- a/packages/backend-common/src/reading/FetchUrlReader.ts +++ b/packages/backend-common/src/reading/FetchUrlReader.ts @@ -24,6 +24,7 @@ import { SearchResponse, UrlReader, } from './types'; +import { normalize as normalizePath } from 'path'; /** * A UrlReader that does a plain fetch of the URL. @@ -39,18 +40,28 @@ export class FetchUrlReader implements UrlReader { * `host`: * Either full hostnames to match, or subdomain wildcard matchers with a leading `*`. * For example `example.com` and `*.example.com` are valid values, `prod.*.example.com` is not. + * + * `paths`: + * An optional list of paths which are allowed. If the list is omitted all paths are allowed. */ static factory: ReaderFactory = ({ config }) => { const predicates = config .getOptionalConfigArray('backend.reading.allow') ?.map(allowConfig => { + const paths = allowConfig.getOptionalStringArray('paths'); + const checkPath = paths + ? (url: URL) => { + const targetPath = normalizePath(url.pathname); + return paths.some(path => targetPath.startsWith(path)); + } + : (_url: URL) => true; const host = allowConfig.getString('host'); if (host.startsWith('*.')) { const suffix = host.slice(1); - return (url: URL) => url.host.endsWith(suffix); + return (url: URL) => url.host.endsWith(suffix) && checkPath(url); } - return (url: URL) => url.host === host; + return (url: URL) => url.host === host && checkPath(url); }) ?? []; const reader = new FetchUrlReader(); From bfc837a97b5cd16916ac241067df90e62a149479 Mon Sep 17 00:00:00 2001 From: Gabriele Mambrini Date: Fri, 5 Nov 2021 09:53:45 +0100 Subject: [PATCH 004/505] Update docs for allowed paths in backend.reading.allow Signed-off-by: Gabriele Mambrini --- docs/features/software-catalog/descriptor-format.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/features/software-catalog/descriptor-format.md b/docs/features/software-catalog/descriptor-format.md index 7100e86ed0..57473ce4c7 100644 --- a/docs/features/software-catalog/descriptor-format.md +++ b/docs/features/software-catalog/descriptor-format.md @@ -144,7 +144,8 @@ spec: Note that to be able to read from targets that are outside of the normal integration points such as `github.com`, you'll need to explicitly allow it by -adding an entry in the `backend.reading.allow` list. For example: +adding an entry in the `backend.reading.allow` list. Paths can be specified to +further restrict targets For example: ```yml backend: @@ -153,6 +154,8 @@ backend: allow: - host: example.com - host: '*.examples.org' + - host: example.net + paths: ['/api/'] ``` ## Common to All Kinds: The Envelope From 1daada3a06f511ee31614822e7d00cd4f88fd0fe Mon Sep 17 00:00:00 2001 From: Gabriele Mambrini Date: Fri, 5 Nov 2021 10:00:17 +0100 Subject: [PATCH 005/505] Added changeset Signed-off-by: Gabriele Mambrini --- .changeset/nasty-impalas-travel.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/nasty-impalas-travel.md diff --git a/.changeset/nasty-impalas-travel.md b/.changeset/nasty-impalas-travel.md new file mode 100644 index 0000000000..f6eb321628 --- /dev/null +++ b/.changeset/nasty-impalas-travel.md @@ -0,0 +1,5 @@ +--- +'@backstage/backend-common': patch +--- + +Paths can be specified in backend.reading.allow to further restrict allowed targets From fb7a0bdeac53d15c02e472e3eaebeb9b16c2aad3 Mon Sep 17 00:00:00 2001 From: Pavel Date: Thu, 28 Oct 2021 10:48:51 +0200 Subject: [PATCH 006/505] Update README.md This is needed, otherwise, after you're redirected back there is an error from browser and nothing works. Signed-off-by: Pavlo Poliakov --- plugins/api-docs/README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/plugins/api-docs/README.md b/plugins/api-docs/README.md index d5ed198dc1..be98304cfe 100644 --- a/plugins/api-docs/README.md +++ b/plugins/api-docs/README.md @@ -160,6 +160,18 @@ by this plugin. Grab a copy of [oauth2-redirect.html](https://github.com/swagger-api/swagger-ui/blob/master/dist/oauth2-redirect.html) and put it in the `app/public/` directory in order to enable Swagger UI to complete this redirection. +This also may require you to adjust `Content Security Policy` header settings of your backstage application. So javascript on `oauth2-redirect.html` can be executed. + +There are two steps: +1. Open `oauth2-redirect.html` for editing and add `nonce` to the `script` tag. Like this + ``` +