Merge pull request #13141 from darthaud/fix/allow-skip-migrations-all-plugins

Fix/allow skip migrations all plugins
This commit is contained in:
Patrik Oldsberg
2022-08-18 16:26:01 +02:00
committed by GitHub
26 changed files with 294 additions and 94 deletions
+16
View File
@@ -0,0 +1,16 @@
---
'@backstage/plugin-scaffolder-backend': minor
---
Fixed a bug in plugin-scaffolder-backend where it ignores the skip migration database options.
To use this new implementation you need to create the instance of `DatabaseTaskStore` using the `PluginDatabaseManager` instead of `Knex`;
```
import { DatabaseManager, getRootLogger, loadBackendConfig } from '@backstage/backend-common';
import { DatabaseTaskStore } from '@backstage/plugin-scaffolder-backend';
const config = await loadBackendConfig({ argv: process.argv, logger: getRootLogger() });
const databaseManager = DatabaseManager.fromConfig(config, { migrations: { skip: true } });
const databaseTaskStore = await DatabaseTaskStore.create(databaseManager);
```
+16
View File
@@ -0,0 +1,16 @@
---
'@backstage/plugin-search-backend-module-pg': minor
---
Fixed a bug in search-backend-module-pg where it ignores the skip migration database options when using the database.
To use this new implementation you need to create the instance of `DatabaseDocumentStore` using the `PluginDatabaseManager` instead of `Knex`;
```
import { DatabaseManager, getRootLogger, loadBackendConfig } from '@backstage/backend-common';
import { DatabaseDocumentStore } from '@backstage/plugin-search-backend-module-pg';
const config = await loadBackendConfig({ argv: process.argv, logger: getRootLogger() });
const databaseManager = DatabaseManager.fromConfig(config, { migrations: { skip: true } });
const databaseDocumentStore = await DatabaseDocumentStore.create(databaseManager);
```
+9
View File
@@ -0,0 +1,9 @@
---
'@backstage/backend-tasks': patch
'@backstage/plugin-app-backend': patch
'@backstage/plugin-bazaar-backend': patch
'@backstage/plugin-code-coverage-backend': patch
'@backstage/plugin-tech-insights-backend': patch
---
Fixed a bug where the database option to skip migrations was ignored.
@@ -58,9 +58,12 @@ export class TaskScheduler {
*/
forPlugin(pluginId: string): PluginTaskScheduler {
const databaseFactory = once(async () => {
const knex = await this.databaseManager.forPlugin(pluginId).getClient();
const databaseManager = this.databaseManager.forPlugin(pluginId);
const knex = await databaseManager.getClient();
await migrateBackendTasks(knex);
if (!databaseManager.migrations?.skip) {
await migrateBackendTasks(knex);
}
const janitor = new PluginTaskSchedulerJanitor({
knex,
@@ -14,12 +14,25 @@
* limitations under the License.
*/
import { Knex as KnexType } from 'knex';
import { getVoidLogger } from '@backstage/backend-common';
import { TestDatabases } from '@backstage/backend-test-utils';
import { StaticAssetsStore } from './StaticAssetsStore';
const logger = getVoidLogger();
function createDatabaseManager(
client: KnexType,
skipMigrations: boolean = false,
) {
return {
getClient: async () => client,
migrations: {
skip: skipMigrations,
},
};
}
describe('StaticAssetsStore', () => {
const databases = TestDatabases.create({
ids: ['POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'],
@@ -28,9 +41,11 @@ describe('StaticAssetsStore', () => {
it.each(databases.eachSupportedId())(
'should store and retrieve assets, %p',
async databaseId => {
const client = await databases.init(databaseId);
const database = createDatabaseManager(client);
const store = await StaticAssetsStore.create({
logger,
database: await databases.init(databaseId),
database,
});
await store.storeAssets([
@@ -69,9 +84,11 @@ describe('StaticAssetsStore', () => {
it.each(databases.eachSupportedId())(
'should update assets timestamps, but not contents, %p',
async databaseId => {
const client = await databases.init(databaseId);
const database = createDatabaseManager(client);
const store = await StaticAssetsStore.create({
logger,
database: await databases.init(databaseId),
database,
});
await store.storeAssets([
@@ -119,7 +136,8 @@ describe('StaticAssetsStore', () => {
it.each(databases.eachSupportedId())(
'should trim old assets, %p',
async databaseId => {
const database = await databases.init(databaseId);
const knex = await databases.init(databaseId);
const database = createDatabaseManager(knex);
const store = await StaticAssetsStore.create({
logger,
database,
@@ -137,12 +155,12 @@ describe('StaticAssetsStore', () => {
]);
// Rewrite modified time of "old" to be 1h in the past
const updated = await database('static_assets_cache')
const updated = await knex('static_assets_cache')
.where({ path: 'old' })
.update({
last_modified_at: database.client.config.client.includes('sqlite3')
? database.raw(`datetime('now', '-3600 seconds')`)
: database.raw(`now() + interval '-3600 seconds'`),
last_modified_at: knex.client.config.client.includes('sqlite3')
? knex.raw(`datetime('now', '-3600 seconds')`)
: knex.raw(`now() + interval '-3600 seconds'`),
});
expect(updated).toBe(1);
@@ -14,7 +14,10 @@
* limitations under the License.
*/
import { resolvePackagePath } from '@backstage/backend-common';
import {
PluginDatabaseManager,
resolvePackagePath,
} from '@backstage/backend-common';
import { Knex } from 'knex';
import { Logger } from 'winston';
import { DateTime } from 'luxon';
@@ -34,7 +37,7 @@ interface StaticAssetRow {
/** @internal */
export interface StaticAssetsStoreOptions {
database: Knex;
database: PluginDatabaseManager;
logger: Logger;
}
@@ -48,15 +51,21 @@ export class StaticAssetsStore implements StaticAssetProvider {
#logger: Logger;
static async create(options: StaticAssetsStoreOptions) {
await options.database.migrate.latest({
directory: migrationsDir,
});
return new StaticAssetsStore(options);
const { database } = options;
const client = await database.getClient();
if (!database.migrations?.skip) {
await client.migrate.latest({
directory: migrationsDir,
});
}
return new StaticAssetsStore(client, options.logger);
}
private constructor(options: StaticAssetsStoreOptions) {
this.#db = options.database;
this.#logger = options.logger;
private constructor(client: Knex, logger: Logger) {
this.#db = client;
this.#logger = logger;
}
/**
+1 -1
View File
@@ -130,7 +130,7 @@ export async function createRouter(
if (options.database) {
const store = await StaticAssetsStore.create({
logger,
database: await options.database.getClient(),
database: options.database,
});
const assets = await findStaticAssets(staticDir);
@@ -16,6 +16,7 @@
import { DatabaseHandler } from './DatabaseHandler';
import { TestDatabaseId, TestDatabases } from '@backstage/backend-test-utils';
import { Knex as KnexType } from 'knex';
const bazaarProject: any = {
name: 'n1',
@@ -35,11 +36,24 @@ describe('DatabaseHandler', () => {
ids: ['POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'],
});
function createDatabaseManager(
client: KnexType,
skipMigrations: boolean = false,
) {
return {
getClient: async () => client,
migrations: {
skip: skipMigrations,
},
};
}
async function createDatabaseHandler(databaseId: TestDatabaseId) {
const knex = await databases.init(databaseId);
const databaseManager = createDatabaseManager(knex);
return {
knex,
dbHandler: await DatabaseHandler.create({ database: knex }),
dbHandler: await DatabaseHandler.create({ database: databaseManager }),
};
}
@@ -14,7 +14,10 @@
* limitations under the License.
*/
import { resolvePackagePath } from '@backstage/backend-common';
import {
PluginDatabaseManager,
resolvePackagePath,
} from '@backstage/backend-common';
import { Knex } from 'knex';
const migrationsDir = resolvePackagePath(
@@ -23,24 +26,27 @@ const migrationsDir = resolvePackagePath(
);
type Options = {
database: Knex;
database: PluginDatabaseManager;
};
export class DatabaseHandler {
static async create(options: Options): Promise<DatabaseHandler> {
const { database } = options;
const client = await database.getClient();
await database.migrate.latest({
directory: migrationsDir,
});
if (!database.migrations?.skip) {
await client.migrate.latest({
directory: migrationsDir,
});
}
return new DatabaseHandler(options);
return new DatabaseHandler(client);
}
private readonly database: Knex;
private readonly client: Knex;
private constructor(options: Options) {
this.database = options.database;
private constructor(client: Knex) {
this.client = client;
}
private columns = [
@@ -58,14 +64,11 @@ export class DatabaseHandler {
];
async getMembers(id: string) {
return await this.database
.select('*')
.from('members')
.where({ item_id: id });
return await this.client.select('*').from('members').where({ item_id: id });
}
async addMember(id: number, userId: string, picture?: string) {
await this.database
await this.client
.insert({
item_id: id,
user_id: userId,
@@ -75,18 +78,18 @@ export class DatabaseHandler {
}
async deleteMember(id: number, userId: string) {
return await this.database('members')
return await this.client('members')
.where({ item_id: id })
.andWhere('user_id', userId)
.del();
}
async getMetadataById(id: number) {
const coalesce = this.database.raw(
const coalesce = this.client.raw(
'coalesce(count(members.item_id), 0) as members_count',
);
return await this.database('metadata')
return await this.client('metadata')
.select([...this.columns, coalesce])
.where({ 'metadata.id': id })
.groupBy(this.columns)
@@ -94,11 +97,11 @@ export class DatabaseHandler {
}
async getMetadataByRef(entityRef: string) {
const coalesce = this.database.raw(
const coalesce = this.client.raw(
'coalesce(count(members.item_id), 0) as members_count',
);
return await this.database('metadata')
return await this.client('metadata')
.select([...this.columns, coalesce])
.where({ 'metadata.entity_ref': entityRef })
.groupBy(this.columns)
@@ -118,7 +121,7 @@ export class DatabaseHandler {
responsible,
} = bazaarProject;
await this.database
await this.client
.insert({
name,
entity_ref: entityRef,
@@ -148,7 +151,7 @@ export class DatabaseHandler {
responsible,
} = bazaarProject;
return await this.database('metadata').where({ id: id }).update({
return await this.client('metadata').where({ id: id }).update({
name,
entity_ref: entityRef,
description,
@@ -163,15 +166,15 @@ export class DatabaseHandler {
}
async deleteMetadata(id: number) {
return await this.database('metadata').where({ id: id }).del();
return await this.client('metadata').where({ id: id }).del();
}
async getProjects() {
const coalesce = this.database.raw(
const coalesce = this.client.raw(
'coalesce(count(members.item_id), 0) as members_count',
);
return await this.database('metadata')
return await this.client('metadata')
.select([...this.columns, coalesce])
.groupBy(this.columns)
.leftJoin('members', 'metadata.id', '=', 'members.item_id');
+1 -2
View File
@@ -33,9 +33,8 @@ export async function createRouter(
options: RouterOptions,
): Promise<express.Router> {
const { logger, database } = options;
const db = await database.getClient();
const dbHandler = await DatabaseHandler.create({ database: db });
const dbHandler = await DatabaseHandler.create({ database });
logger.info('Initializing Bazaar backend');
@@ -13,6 +13,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Knex as KnexType } from 'knex';
import { DatabaseManager } from '@backstage/backend-common';
import { stringifyEntityRef } from '@backstage/catalog-model';
import { ConfigReader } from '@backstage/config';
@@ -95,12 +96,24 @@ const coverage: Array<JsonCodeCoverage> = [
],
},
];
function createDatabaseManager(
client: KnexType,
skipMigrations: boolean = false,
) {
return {
getClient: async () => client,
migrations: {
skip: skipMigrations,
},
};
}
let database: CodeCoverageStore;
describe('CodeCoverageDatabase', () => {
beforeAll(async () => {
const client = await db.getClient();
database = await CodeCoverageDatabase.create(client);
const databaseManager = createDatabaseManager(client);
database = await CodeCoverageDatabase.create(databaseManager);
await database.insertCodeCoverage(coverage[0]);
await database.insertCodeCoverage(coverage[1]);
});
@@ -13,7 +13,10 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { resolvePackagePath } from '@backstage/backend-common';
import {
PluginDatabaseManager,
resolvePackagePath,
} from '@backstage/backend-common';
import { NotFoundError } from '@backstage/errors';
import { parseEntityRef, stringifyEntityRef } from '@backstage/catalog-model';
import { Knex } from 'knex';
@@ -41,10 +44,17 @@ const migrationsDir = resolvePackagePath(
);
export class CodeCoverageDatabase implements CodeCoverageStore {
static async create(knex: Knex): Promise<CodeCoverageStore> {
await knex.migrate.latest({
directory: migrationsDir,
});
static async create(
database: PluginDatabaseManager,
): Promise<CodeCoverageStore> {
const knex = await database.getClient();
if (!database.migrations?.skip) {
await knex.migrate.latest({
directory: migrationsDir,
});
}
return new CodeCoverageDatabase(knex);
}
@@ -57,9 +57,7 @@ export const makeRouter = async (
): Promise<express.Router> => {
const { config, logger, discovery, database, urlReader } = options;
const codeCoverageDatabase = await CodeCoverageDatabase.create(
await database.getClient(),
);
const codeCoverageDatabase = await CodeCoverageDatabase.create(database);
const codecovUrl = await discovery.getExternalBaseUrl('code-coverage');
const catalogApi: CatalogApi = new CatalogClient({ discoveryApi: discovery });
const scm = ScmIntegrations.fromConfig(config);
+1 -1
View File
@@ -501,7 +501,7 @@ export class DatabaseTaskStore implements TaskStore {
// @public
export type DatabaseTaskStoreOptions = {
database: Knex;
database: PluginDatabaseManager | Knex;
};
// @public
@@ -15,7 +15,10 @@
*/
import { JsonObject } from '@backstage/types';
import { resolvePackagePath } from '@backstage/backend-common';
import {
PluginDatabaseManager,
resolvePackagePath,
} from '@backstage/backend-common';
import { ConflictError, NotFoundError } from '@backstage/errors';
import { Knex } from 'knex';
import { v4 as uuid } from 'uuid';
@@ -61,9 +64,20 @@ export type RawDbTaskEventRow = {
* @public
*/
export type DatabaseTaskStoreOptions = {
database: Knex;
database: PluginDatabaseManager | Knex;
};
/**
* Typeguard to help DatabaseTaskStore understand when database is PluginDatabaseManager vs. when database is a Knex instance.
*
* * @public
*/
function isPluginDatabaseManager(
opt: PluginDatabaseManager | Knex,
): opt is PluginDatabaseManager {
return (opt as PluginDatabaseManager).getClient !== undefined;
}
const parseSqlDateToIsoString = <T>(input: T): T | string => {
if (typeof input === 'string') {
return DateTime.fromSQL(input, { zone: 'UTC' }).toISO();
@@ -83,14 +97,45 @@ export class DatabaseTaskStore implements TaskStore {
static async create(
options: DatabaseTaskStoreOptions,
): Promise<DatabaseTaskStore> {
await options.database.migrate.latest({
directory: migrationsDir,
});
return new DatabaseTaskStore(options);
const { database } = options;
const client = await this.getClient(database);
await this.runMigrations(database, client);
return new DatabaseTaskStore(client);
}
private constructor(options: DatabaseTaskStoreOptions) {
this.db = options.database;
private static async getClient(
database: PluginDatabaseManager | Knex,
): Promise<Knex> {
if (isPluginDatabaseManager(database)) {
return database.getClient();
}
return database;
}
private static async runMigrations(
database: PluginDatabaseManager | Knex,
client: Knex,
): Promise<void> {
if (!isPluginDatabaseManager(database)) {
await client.migrate.latest({
directory: migrationsDir,
});
return;
}
if (!database.migrations?.skip) {
await client.migrate.latest({
directory: migrationsDir,
});
}
}
private constructor(client: Knex) {
this.db = client;
}
async list(options: {
@@ -32,8 +32,9 @@ async function createStore(): Promise<DatabaseTaskStore> {
},
}),
).forPlugin('scaffolder');
return await DatabaseTaskStore.create({
database: await manager.getClient(),
database: manager,
});
}
@@ -40,7 +40,7 @@ async function createStore(): Promise<DatabaseTaskStore> {
}),
).forPlugin('scaffolder');
return await DatabaseTaskStore.create({
database: await manager.getClient(),
database: manager,
});
}
@@ -127,7 +127,7 @@ describe('createRouter', () => {
beforeEach(async () => {
const logger = getVoidLogger();
const databaseTaskStore = await DatabaseTaskStore.create({
database: await createDatabase().getClient(),
database: createDatabase(),
});
taskBroker = new StorageTaskBroker(databaseTaskStore, logger);
@@ -97,9 +97,7 @@ export async function createRouter(
let taskBroker: TaskBroker;
if (!options.taskBroker) {
const databaseTaskStore = await DatabaseTaskStore.create({
database: await database.getClient(),
});
const databaseTaskStore = await DatabaseTaskStore.create({ database });
taskBroker = new StorageTaskBroker(databaseTaskStore, logger);
} else {
taskBroker = options.taskBroker;
@@ -26,7 +26,9 @@ export class DatabaseDocumentStore implements DatabaseStore {
// (undocumented)
completeInsert(tx: Knex.Transaction, type: string): Promise<void>;
// (undocumented)
static create(knex: Knex): Promise<DatabaseDocumentStore>;
static create(
database: PluginDatabaseManager,
): Promise<DatabaseDocumentStore>;
// (undocumented)
getTransaction(): Promise<Knex.Transaction>;
// (undocumented)
@@ -115,14 +115,14 @@ export class PgSearchEngine implements SearchEngine {
config: Config;
}): Promise<PgSearchEngine> {
return new PgSearchEngine(
await DatabaseDocumentStore.create(await options.database.getClient()),
await DatabaseDocumentStore.create(options.database),
options.config,
);
}
static async fromConfig(config: Config, options: PgSearchOptions) {
return new PgSearchEngine(
await DatabaseDocumentStore.create(await options.database.getClient()),
await DatabaseDocumentStore.create(options.database),
config,
);
}
@@ -13,6 +13,8 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Knex as KnexType } from 'knex';
import { TestDatabaseId, TestDatabases } from '@backstage/backend-test-utils';
import { IndexableDocument } from '@backstage/plugin-search-common';
import { PgSearchHighlightOptions } from '../PgSearchEngine';
@@ -30,6 +32,18 @@ const highlightOptions: PgSearchHighlightOptions = {
fragmentDelimiter: ' ... ',
};
function createDatabaseManager(
client: KnexType,
skipMigrations: boolean = false,
) {
return {
getClient: async () => client,
migrations: {
skip: skipMigrations,
},
};
}
describe('DatabaseDocumentStore', () => {
describe('unsupported', () => {
const databases = TestDatabases.create({
@@ -51,9 +65,10 @@ describe('DatabaseDocumentStore', () => {
'should fail to create, %p',
async databaseId => {
const knex = await databases.init(databaseId);
const databaseManager = createDatabaseManager(knex);
await expect(
async () => await DatabaseDocumentStore.create(knex),
async () => await DatabaseDocumentStore.create(databaseManager),
).rejects.toThrow();
},
60_000,
@@ -67,7 +82,9 @@ describe('DatabaseDocumentStore', () => {
async function createStore(databaseId: TestDatabaseId) {
const knex = await databases.init(databaseId);
const store = await DatabaseDocumentStore.create(knex);
const databaseManager = createDatabaseManager(knex);
const store = await DatabaseDocumentStore.create(databaseManager);
return { store, knex };
}
@@ -13,7 +13,10 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { resolvePackagePath } from '@backstage/backend-common';
import {
PluginDatabaseManager,
resolvePackagePath,
} from '@backstage/backend-common';
import { IndexableDocument } from '@backstage/plugin-search-common';
import { Knex } from 'knex';
import {
@@ -30,7 +33,10 @@ const migrationsDir = resolvePackagePath(
);
export class DatabaseDocumentStore implements DatabaseStore {
static async create(knex: Knex): Promise<DatabaseDocumentStore> {
static async create(
database: PluginDatabaseManager,
): Promise<DatabaseDocumentStore> {
const knex = await database.getClient();
try {
const majorVersion = await queryPostgresMajorVersion(knex);
@@ -49,9 +55,12 @@ export class DatabaseDocumentStore implements DatabaseStore {
);
}
await knex.migrate.latest({
directory: migrationsDir,
});
if (!database.migrations?.skip) {
await knex.migrate.latest({
directory: migrationsDir,
});
}
return new DatabaseDocumentStore(knex);
}
@@ -15,7 +15,7 @@
*/
import { DateTime, Duration } from 'luxon';
import { TechInsightsStore } from '@backstage/plugin-tech-insights-node';
import { Knex } from 'knex';
import { Knex as KnexType, Knex } from 'knex';
import { TestDatabases } from '@backstage/backend-test-utils';
import { getVoidLogger } from '@backstage/backend-common';
import { initializePersistenceContext } from './persistenceContext';
@@ -165,15 +165,28 @@ const multipleSameFacts = [
},
];
function createDatabaseManager(
client: KnexType,
skipMigrations: boolean = false,
) {
return {
getClient: async () => client,
migrations: {
skip: skipMigrations,
},
};
}
describe('Tech Insights database', () => {
const databases = TestDatabases.create();
let store: TechInsightsStore;
let testDbClient: Knex<any, unknown[]>;
beforeAll(async () => {
testDbClient = await databases.init('SQLITE_3');
const database = createDatabaseManager(testDbClient);
store = (
await initializePersistenceContext(testDbClient, {
await initializePersistenceContext(database, {
logger: getVoidLogger(),
})
).techInsightsStore;
@@ -13,8 +13,11 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { getVoidLogger, resolvePackagePath } from '@backstage/backend-common';
import { Knex } from 'knex';
import {
getVoidLogger,
PluginDatabaseManager,
resolvePackagePath,
} from '@backstage/backend-common';
import { Logger } from 'winston';
import { TechInsightsDatabase } from './TechInsightsDatabase';
import { TechInsightsStore } from '@backstage/plugin-tech-insights-node';
@@ -47,13 +50,18 @@ const defaultOptions: CreateDatabaseOptions = {
* @public
*/
export const initializePersistenceContext = async (
knex: Knex,
database: PluginDatabaseManager,
options: CreateDatabaseOptions = defaultOptions,
): Promise<PersistenceContext> => {
await knex.migrate.latest({
directory: migrationsDir,
});
const client = await database.getClient();
if (!database.migrations?.skip) {
await client.migrate.latest({
directory: migrationsDir,
});
}
return {
techInsightsStore: new TechInsightsDatabase(knex, options.logger),
techInsightsStore: new TechInsightsDatabase(client, options.logger),
};
};
@@ -135,10 +135,9 @@ export const buildTechInsightsContext = async <
const factRetrieverRegistry = buildFactRetrieverRegistry();
const persistenceContext = await initializePersistenceContext(
await database.getClient(),
{ logger },
);
const persistenceContext = await initializePersistenceContext(database, {
logger,
});
const factRetrieverEngine = await FactRetrieverEngine.create({
scheduler,