diff --git a/.changeset/purple-jokes-rhyme.md b/.changeset/purple-jokes-rhyme.md new file mode 100644 index 0000000000..055a192c4d --- /dev/null +++ b/.changeset/purple-jokes-rhyme.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +Add a base plate for performance testing of the catalog diff --git a/plugins/catalog-backend/src/tests/README.md b/plugins/catalog-backend/src/tests/README.md new file mode 100644 index 0000000000..8f4dce6379 --- /dev/null +++ b/plugins/catalog-backend/src/tests/README.md @@ -0,0 +1,3 @@ +# Overall Tests + +These are tests that do not apply to any one single class, and/or aren't regular unit tests. diff --git a/plugins/catalog-backend/src/integration.test.ts b/plugins/catalog-backend/src/tests/integration.test.ts similarity index 94% rename from plugins/catalog-backend/src/integration.test.ts rename to plugins/catalog-backend/src/tests/integration.test.ts index 9aae415829..e7f983f51f 100644 --- a/plugins/catalog-backend/src/integration.test.ts +++ b/plugins/catalog-backend/src/tests/integration.test.ts @@ -14,45 +14,46 @@ * limitations under the License. */ -import { Knex } from 'knex'; -import { Logger } from 'winston'; -import { ConfigReader } from '@backstage/config'; -import { JsonObject } from '@backstage/types'; -import { CatalogProcessingEngine, EntityProvider } from './index'; import { DatabaseManager, getVoidLogger } from '@backstage/backend-common'; -import { PermissionEvaluator } from '@backstage/plugin-permission-common'; import { Entity, EntityPolicies, stringifyEntityRef, } from '@backstage/catalog-model'; -import { defaultEntityDataParser } from './modules/util/parse'; -import { DefaultCatalogProcessingOrchestrator } from './processing/DefaultCatalogProcessingOrchestrator'; -import { applyDatabaseMigrations } from './database/migrations'; -import { DefaultCatalogDatabase } from './database/DefaultCatalogDatabase'; -import { DefaultProcessingDatabase } from './database/DefaultProcessingDatabase'; +import { ConfigReader } from '@backstage/config'; +import { InputError } from '@backstage/errors'; import { ScmIntegrations } from '@backstage/integration'; -import { DefaultCatalogRulesEnforcer } from './ingestion/CatalogRules'; -import { Stitcher } from './stitching/Stitcher'; -import { DefaultEntitiesCatalog } from './service/DefaultEntitiesCatalog'; +import { LocationSpec } from '@backstage/plugin-catalog-common'; +import { + CatalogProcessorEmit, + EntityProvider, + EntityProviderConnection, + processingResult, +} from '@backstage/plugin-catalog-node'; +import { PermissionEvaluator } from '@backstage/plugin-permission-common'; +import { JsonObject } from '@backstage/types'; +import { createHash } from 'crypto'; +import { Knex } from 'knex'; +import { Logger } from 'winston'; +import { EntitiesCatalog } from '../catalog/types'; +import { DefaultCatalogDatabase } from '../database/DefaultCatalogDatabase'; +import { DefaultProcessingDatabase } from '../database/DefaultProcessingDatabase'; +import { DefaultProviderDatabase } from '../database/DefaultProviderDatabase'; +import { applyDatabaseMigrations } from '../database/migrations'; +import { RefreshStateItem } from '../database/types'; +import { DefaultCatalogRulesEnforcer } from '../ingestion/CatalogRules'; +import { defaultEntityDataParser } from '../modules/util/parse'; import { DefaultCatalogProcessingEngine, ProgressTracker, -} from './processing/DefaultCatalogProcessingEngine'; -import { createHash } from 'crypto'; -import { DefaultRefreshService } from './service/DefaultRefreshService'; -import { connectEntityProviders } from './processing/connectEntityProviders'; -import { EntitiesCatalog } from './catalog/types'; -import { RefreshOptions, RefreshService } from './service/types'; -import { - CatalogProcessorEmit, - EntityProviderConnection, - LocationSpec, - processingResult, -} from '@backstage/plugin-catalog-node'; -import { RefreshStateItem } from './database/types'; -import { DefaultProviderDatabase } from './database/DefaultProviderDatabase'; -import { InputError } from '@backstage/errors'; +} from '../processing/DefaultCatalogProcessingEngine'; +import { DefaultCatalogProcessingOrchestrator } from '../processing/DefaultCatalogProcessingOrchestrator'; +import { connectEntityProviders } from '../processing/connectEntityProviders'; +import { CatalogProcessingEngine } from '../processing/types'; +import { DefaultEntitiesCatalog } from '../service/DefaultEntitiesCatalog'; +import { DefaultRefreshService } from '../service/DefaultRefreshService'; +import { RefreshOptions, RefreshService } from '../service/types'; +import { Stitcher } from '../stitching/Stitcher'; const voidLogger = getVoidLogger(); diff --git a/plugins/catalog-backend/src/migrations.test.ts b/plugins/catalog-backend/src/tests/migrations.test.ts similarity index 99% rename from plugins/catalog-backend/src/migrations.test.ts rename to plugins/catalog-backend/src/tests/migrations.test.ts index 779d2582bd..d2c00e31c0 100644 --- a/plugins/catalog-backend/src/migrations.test.ts +++ b/plugins/catalog-backend/src/tests/migrations.test.ts @@ -18,7 +18,7 @@ import { Knex } from 'knex'; import { TestDatabases } from '@backstage/backend-test-utils'; import fs from 'fs'; -const migrationsDir = `${__dirname}/../migrations`; +const migrationsDir = `${__dirname}/../../migrations`; const migrationsFiles = fs.readdirSync(migrationsDir).sort(); async function migrateUpOnce(knex: Knex): Promise { diff --git a/plugins/catalog-backend/src/tests/performance/README.md b/plugins/catalog-backend/src/tests/performance/README.md new file mode 100644 index 0000000000..c1a12b1f71 --- /dev/null +++ b/plugins/catalog-backend/src/tests/performance/README.md @@ -0,0 +1,6 @@ +# Catalog Performance Tests + +These are regular jest tests, but they are only enabled when the +`PERFORMANCE_TEST` environment variable is set. You can also set the +`PERFORMANCE_TRACE` environment variable to get additional trace output while +the tests run. diff --git a/plugins/catalog-backend/src/tests/performance/lib/catalogModuleSyntheticLoadEntities.ts b/plugins/catalog-backend/src/tests/performance/lib/catalogModuleSyntheticLoadEntities.ts new file mode 100644 index 0000000000..4d38b28bcf --- /dev/null +++ b/plugins/catalog-backend/src/tests/performance/lib/catalogModuleSyntheticLoadEntities.ts @@ -0,0 +1,257 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { createBackendModule } from '@backstage/backend-plugin-api'; +import { Entity } from '@backstage/catalog-model'; +import { LocationSpec } from '@backstage/plugin-catalog-common'; +import { + CatalogProcessor, + CatalogProcessorCache, + CatalogProcessorEmit, + DeferredEntity, + EntityProvider, + EntityProviderConnection, + processingResult, +} from '@backstage/plugin-catalog-node'; +import { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node/alpha'; + +/** + * Options for a fixed initial load of entities. + */ +export type SyntheticLoadOptions = { + /** + * The number of entities to insert. + */ + baseEntitiesCount: number; + /** + * The number of outgoing relations per entity. + */ + baseRelationsCount: number; + /** + * How "bunched up" relations are, as a number between 0 and 1. + * + * 0 means that relations are evenly distributed such that they form a uniform + * mesh. 1 means that every single relation goes to one and the same exact + * "victim" entity. Thus, the higher the number, the more unevenly distributed + * the relations are. + */ + baseRelationsSkew: number; + /** + * The number of child entities emitted by each base entity. + */ + childrenCount: number; +}; + +/** + * Events that can occur during the load ingestion + */ +export type SyntheticLoadEvents = { + onBeforeInsertBaseEntities?: () => void; + onAfterInsertBaseEntities?: () => void; + onError?: (error: Error) => void; +}; + +/** + * Throws if any of the options are invalid. + */ +function validateSyntheticLoadOptions(options: SyntheticLoadOptions): void { + if ( + !Number.isSafeInteger(options.baseEntitiesCount) || + options.baseEntitiesCount <= 0 + ) { + throw new TypeError('baseEntitiesCount must be a nonnegative integer'); + } else if ( + !Number.isSafeInteger(options.baseRelationsCount) || + options.baseRelationsCount < 0 + ) { + throw new TypeError('baseRelationsCount must be a nonnegative integer'); + } else if ( + !Number.isFinite(options.baseRelationsSkew) || + options.baseRelationsSkew < 0 || + options.baseRelationsSkew > 1 + ) { + throw new TypeError('baseRelationsSkew must be a number between 0 and 1'); + } else if ( + !Number.isSafeInteger(options.childrenCount) || + options.childrenCount < 0 + ) { + throw new TypeError('childrenCount must be a nonnegative integer'); + } +} + +/** + * Some shared definitions + */ +export const common = { + baseEntityName: (index: number) => `synthetic-${index}`, + baseEntityNamespace: 'synthetic-load-base', + baseEntity: (index: number): Entity => { + return { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Location', + metadata: { + name: common.baseEntityName(index), + namespace: common.baseEntityNamespace, + annotations: { + 'backstage.io/managed-by-location': `url:fake`, + 'backstage.io/managed-by-origin-location': `url:fake`, + }, + }, + spec: { + type: 'url', + targets: [], + }, + }; + }, + childEntityName: (baseEntity: Entity, index: number) => + `${baseEntity.metadata.name}-${index}`, + childEntityNamespace: 'synthetic-load-child', + childEntity: (baseEntity: Entity, index: number): Entity => { + return { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Location', + metadata: { + name: common.childEntityName(baseEntity, index), + namespace: common.childEntityNamespace, + annotations: { + 'backstage.io/managed-by-location': `url:fake`, + 'backstage.io/managed-by-origin-location': `url:fake`, + }, + }, + spec: { + type: 'url', + targets: [], + }, + }; + }, + kind: 'Location', + relationType: 'loadTest', +} as const; + +/** + * The entity provider that drives the initial base entity injection + */ +class SyntheticLoadEntitiesProvider implements EntityProvider { + constructor( + private readonly load: SyntheticLoadOptions, + private readonly events: SyntheticLoadEvents, + ) {} + + getProviderName(): string { + return 'SyntheticLoadEntitiesProvider'; + } + + async connect(connection: EntityProviderConnection): Promise { + // Defer this work so as to not block startup entirely + setImmediate(async () => { + try { + this.events.onBeforeInsertBaseEntities?.(); + + const deferred: DeferredEntity[] = []; + for (let index = 0; index < this.load.baseEntitiesCount; ++index) { + deferred.push({ entity: common.baseEntity(index) }); + } + + await connection.applyMutation({ + type: 'full', + entities: deferred, + }); + + this.events.onAfterInsertBaseEntities?.(); + } catch (error) { + this.events.onError?.(error); + } + }); + } +} + +/** + * Supporting processor for emitting children and relations + */ +class SyntheticLoadEntitiesProcessor implements CatalogProcessor { + constructor(private readonly load: SyntheticLoadOptions) {} + + getProcessorName(): string { + return 'SyntheticLoadEntitiesProcessor'; + } + + async postProcessEntity( + entity: Entity, + location: LocationSpec, + emit: CatalogProcessorEmit, + _cache: CatalogProcessorCache, + ): Promise { + const { + baseEntitiesCount, + baseRelationsCount, + baseRelationsSkew, + childrenCount, + } = this.load; + + if (entity.metadata.namespace === common.baseEntityNamespace) { + for (let rc = 0; rc < baseRelationsCount; ++rc) { + const relationIndex = Math.floor( + Math.random() * baseEntitiesCount * (1 - baseRelationsSkew), + ); + emit( + processingResult.relation({ + source: { + kind: common.kind, + namespace: common.baseEntityNamespace, + name: entity.metadata.name, + }, + target: { + kind: common.kind, + namespace: common.baseEntityNamespace, + name: common.baseEntityName(relationIndex), + }, + type: common.relationType, + }), + ); + } + + for (let i = 0; i < childrenCount; ++i) { + emit(processingResult.entity(location, common.childEntity(entity, i))); + } + } + + return entity; + } +} + +export const catalogModuleSyntheticLoadEntities = createBackendModule( + (options: { load: SyntheticLoadOptions; events?: SyntheticLoadEvents }) => ({ + moduleId: 'syntheticLoadEntities', + pluginId: 'catalog', + register(reg) { + reg.registerInit({ + deps: { + catalog: catalogProcessingExtensionPoint, + }, + async init({ catalog }) { + const { load, events = {} } = options; + + validateSyntheticLoadOptions(load); + const provider = new SyntheticLoadEntitiesProvider(load, events); + const processor = new SyntheticLoadEntitiesProcessor(load); + + catalog.addEntityProvider(provider); + catalog.addProcessor(processor); + }, + }); + }, + }), +); diff --git a/plugins/catalog-backend/src/tests/performance/lib/env.ts b/plugins/catalog-backend/src/tests/performance/lib/env.ts new file mode 100644 index 0000000000..c272930292 --- /dev/null +++ b/plugins/catalog-backend/src/tests/performance/lib/env.ts @@ -0,0 +1,22 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export const performanceTraceEnabled = !!process.env.PERFORMANCE_TRACE; + +export const describePerformanceTest: jest.Describe = process.env + .PERFORMANCE_TEST + ? describe + : describe.skip; diff --git a/plugins/catalog-backend/src/tests/performance/stitchingPerformance.test.ts b/plugins/catalog-backend/src/tests/performance/stitchingPerformance.test.ts new file mode 100644 index 0000000000..2f32fcd907 --- /dev/null +++ b/plugins/catalog-backend/src/tests/performance/stitchingPerformance.test.ts @@ -0,0 +1,195 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + coreServices, + createServiceFactory, +} from '@backstage/backend-plugin-api'; +import { TestDatabases, startTestBackend } from '@backstage/backend-test-utils'; +import { catalogPlugin } from '@backstage/plugin-catalog-backend/alpha'; +import { Knex } from 'knex'; +import { applyDatabaseMigrations } from '../../database/migrations'; +import { + SyntheticLoadEvents, + SyntheticLoadOptions, + catalogModuleSyntheticLoadEntities, +} from './lib/catalogModuleSyntheticLoadEntities'; +import { describePerformanceTest, performanceTraceEnabled } from './lib/env'; + +function defer() { + let resolve: (value: T | PromiseLike) => void; + let reject: (error?: unknown) => void; + const promise = new Promise((_resolve, _reject) => { + resolve = _resolve; + reject = _reject; + }); + return { promise, resolve: resolve!, reject: reject! }; +} + +const traceLog: typeof console.log = performanceTraceEnabled + ? console.log + : () => {}; + +class Tracker { + private insertBaseEntitiesStart: number | undefined; + private insertBaseEntitiesEnd: number | undefined; + private readonly deferred = defer(); + + constructor( + private readonly knex: Knex, + private readonly load: SyntheticLoadOptions, + ) {} + + events(): SyntheticLoadEvents { + return { + onBeforeInsertBaseEntities: () => { + this.insertBaseEntitiesStart = Date.now(); + traceLog(`Inserting ${this.load.baseEntitiesCount} base entities`); + }, + onAfterInsertBaseEntities: async () => { + this.insertBaseEntitiesEnd = Date.now(); + + const insertDuration = ( + (this.insertBaseEntitiesEnd - this.insertBaseEntitiesStart!) / + 1000 + ).toFixed(1); + traceLog( + `Inserted ${this.load.baseEntitiesCount} base entities in ${insertDuration} seconds`, + ); + + await this.completionPolling(); + + const processingDuration = ( + (Date.now() - this.insertBaseEntitiesEnd) / + 1000 + ).toFixed(1); + traceLog( + `Stitched ${this.load.baseEntitiesCount} entities in ${processingDuration} seconds`, + ); + + this.deferred.resolve(); + }, + onError: error => { + this.deferred.reject(error); + }, + }; + } + + async completion(): Promise { + return this.deferred.promise; + } + + private completionPolling() { + const { baseEntitiesCount, childrenCount } = this.load; + const expectedTotal = baseEntitiesCount + baseEntitiesCount * childrenCount; + + let processedTotal = 0; + let stitchedTotal = 0; + + return new Promise((resolve, reject) => { + const interval = setInterval(async () => { + try { + const processedCount = await this.knex('refresh_state') + .count({ count: '*' }) + .whereNotNull('processed_entity') + .then(rows => Number(rows[0].count)); + + const stitchedCount = await this.knex('final_entities') + .count({ count: '*' }) + .whereNotNull('final_entity') + .then(rows => Number(rows[0].count)); + + const processedDelta = processedCount - processedTotal; + const processedPercent = ( + (processedCount / expectedTotal) * + 100 + ).toFixed(1); + const stitchedDelta = stitchedCount - stitchedTotal; + const stitchedPercent = ( + (stitchedCount / expectedTotal) * + 100 + ).toFixed(1); + + const processedSummary = `${processedCount} (${processedPercent}%, ${processedDelta}/s)`; + const stitchedSummary = `${stitchedCount} (${stitchedPercent}%, ${stitchedDelta}/s)`; + traceLog( + `Processed: ${processedSummary}\nStitched: ${stitchedSummary}`, + ); + + processedTotal = processedCount; + stitchedTotal = stitchedCount; + + if (stitchedCount === expectedTotal) { + clearInterval(interval); + resolve(); + } + } catch (error) { + clearInterval(interval); + reject(error); + } + }, 1000); + }); + } +} + +function staticDatabase(knex: Knex) { + return createServiceFactory({ + service: coreServices.database, + deps: {}, + createRootContext: () => undefined, + factory: () => ({ getClient: async () => knex }), + }); +} + +jest.setTimeout(600_000); + +describePerformanceTest('stitchingPerformance', () => { + const databases = TestDatabases.create({ + ids: [/* 'MYSQL_8', */ 'POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'], + }); + + it.each(databases.eachSupportedId())( + 'runs stitching in immediate mode, %p', + async databaseId => { + const knex = await databases.init(databaseId); + await applyDatabaseMigrations(knex); + + const load: SyntheticLoadOptions = { + baseEntitiesCount: 1000, + baseRelationsCount: 3, + baseRelationsSkew: 0.3, + childrenCount: 3, + }; + + const tracker = new Tracker(knex, load); + + const backend = await startTestBackend({ + services: [staticDatabase(knex)], + features: [ + catalogPlugin(), + catalogModuleSyntheticLoadEntities({ + load, + events: tracker.events(), + }), + ], + }); + + await expect(tracker.completion()).resolves.toBeUndefined(); + await backend.stop(); + await knex.destroy(); + }, + ); +});