Merge pull request #18261 from backstage/freben/catalog-load-test
add performance tests
This commit is contained in:
Fredrik Adelöw
@@ -0,0 +1,5 @@
|
||||
---
|
||||
'@backstage/plugin-catalog-backend': patch
|
||||
---
|
||||
|
||||
Add a base plate for performance testing of the catalog
|
||||
@@ -0,0 +1,3 @@
|
||||
# Overall Tests
|
||||
|
||||
These are tests that do not apply to any one single class, and/or aren't regular unit tests.
|
||||
+30
-29
@@ -14,45 +14,46 @@
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { Knex } from 'knex';
|
||||
import { Logger } from 'winston';
|
||||
import { ConfigReader } from '@backstage/config';
|
||||
import { JsonObject } from '@backstage/types';
|
||||
import { CatalogProcessingEngine, EntityProvider } from './index';
|
||||
import { DatabaseManager, getVoidLogger } from '@backstage/backend-common';
|
||||
import { PermissionEvaluator } from '@backstage/plugin-permission-common';
|
||||
import {
|
||||
Entity,
|
||||
EntityPolicies,
|
||||
stringifyEntityRef,
|
||||
} from '@backstage/catalog-model';
|
||||
import { defaultEntityDataParser } from './modules/util/parse';
|
||||
import { DefaultCatalogProcessingOrchestrator } from './processing/DefaultCatalogProcessingOrchestrator';
|
||||
import { applyDatabaseMigrations } from './database/migrations';
|
||||
import { DefaultCatalogDatabase } from './database/DefaultCatalogDatabase';
|
||||
import { DefaultProcessingDatabase } from './database/DefaultProcessingDatabase';
|
||||
import { ConfigReader } from '@backstage/config';
|
||||
import { InputError } from '@backstage/errors';
|
||||
import { ScmIntegrations } from '@backstage/integration';
|
||||
import { DefaultCatalogRulesEnforcer } from './ingestion/CatalogRules';
|
||||
import { Stitcher } from './stitching/Stitcher';
|
||||
import { DefaultEntitiesCatalog } from './service/DefaultEntitiesCatalog';
|
||||
import { LocationSpec } from '@backstage/plugin-catalog-common';
|
||||
import {
|
||||
CatalogProcessorEmit,
|
||||
EntityProvider,
|
||||
EntityProviderConnection,
|
||||
processingResult,
|
||||
} from '@backstage/plugin-catalog-node';
|
||||
import { PermissionEvaluator } from '@backstage/plugin-permission-common';
|
||||
import { JsonObject } from '@backstage/types';
|
||||
import { createHash } from 'crypto';
|
||||
import { Knex } from 'knex';
|
||||
import { Logger } from 'winston';
|
||||
import { EntitiesCatalog } from '../catalog/types';
|
||||
import { DefaultCatalogDatabase } from '../database/DefaultCatalogDatabase';
|
||||
import { DefaultProcessingDatabase } from '../database/DefaultProcessingDatabase';
|
||||
import { DefaultProviderDatabase } from '../database/DefaultProviderDatabase';
|
||||
import { applyDatabaseMigrations } from '../database/migrations';
|
||||
import { RefreshStateItem } from '../database/types';
|
||||
import { DefaultCatalogRulesEnforcer } from '../ingestion/CatalogRules';
|
||||
import { defaultEntityDataParser } from '../modules/util/parse';
|
||||
import {
|
||||
DefaultCatalogProcessingEngine,
|
||||
ProgressTracker,
|
||||
} from './processing/DefaultCatalogProcessingEngine';
|
||||
import { createHash } from 'crypto';
|
||||
import { DefaultRefreshService } from './service/DefaultRefreshService';
|
||||
import { connectEntityProviders } from './processing/connectEntityProviders';
|
||||
import { EntitiesCatalog } from './catalog/types';
|
||||
import { RefreshOptions, RefreshService } from './service/types';
|
||||
import {
|
||||
CatalogProcessorEmit,
|
||||
EntityProviderConnection,
|
||||
LocationSpec,
|
||||
processingResult,
|
||||
} from '@backstage/plugin-catalog-node';
|
||||
import { RefreshStateItem } from './database/types';
|
||||
import { DefaultProviderDatabase } from './database/DefaultProviderDatabase';
|
||||
import { InputError } from '@backstage/errors';
|
||||
} from '../processing/DefaultCatalogProcessingEngine';
|
||||
import { DefaultCatalogProcessingOrchestrator } from '../processing/DefaultCatalogProcessingOrchestrator';
|
||||
import { connectEntityProviders } from '../processing/connectEntityProviders';
|
||||
import { CatalogProcessingEngine } from '../processing/types';
|
||||
import { DefaultEntitiesCatalog } from '../service/DefaultEntitiesCatalog';
|
||||
import { DefaultRefreshService } from '../service/DefaultRefreshService';
|
||||
import { RefreshOptions, RefreshService } from '../service/types';
|
||||
import { Stitcher } from '../stitching/Stitcher';
|
||||
|
||||
const voidLogger = getVoidLogger();
|
||||
|
||||
+1
-1
@@ -18,7 +18,7 @@ import { Knex } from 'knex';
|
||||
import { TestDatabases } from '@backstage/backend-test-utils';
|
||||
import fs from 'fs';
|
||||
|
||||
const migrationsDir = `${__dirname}/../migrations`;
|
||||
const migrationsDir = `${__dirname}/../../migrations`;
|
||||
const migrationsFiles = fs.readdirSync(migrationsDir).sort();
|
||||
|
||||
async function migrateUpOnce(knex: Knex): Promise<void> {
|
||||
@@ -0,0 +1,6 @@
|
||||
# Catalog Performance Tests
|
||||
|
||||
These are regular jest tests, but they are only enabled when the
|
||||
`PERFORMANCE_TEST` environment variable is set. You can also set the
|
||||
`PERFORMANCE_TRACE` environment variable to get additional trace output while
|
||||
the tests run.
|
||||
+257
@@ -0,0 +1,257 @@
|
||||
/*
|
||||
* Copyright 2023 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { createBackendModule } from '@backstage/backend-plugin-api';
|
||||
import { Entity } from '@backstage/catalog-model';
|
||||
import { LocationSpec } from '@backstage/plugin-catalog-common';
|
||||
import {
|
||||
CatalogProcessor,
|
||||
CatalogProcessorCache,
|
||||
CatalogProcessorEmit,
|
||||
DeferredEntity,
|
||||
EntityProvider,
|
||||
EntityProviderConnection,
|
||||
processingResult,
|
||||
} from '@backstage/plugin-catalog-node';
|
||||
import { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node/alpha';
|
||||
|
||||
/**
|
||||
* Options for a fixed initial load of entities.
|
||||
*/
|
||||
export type SyntheticLoadOptions = {
|
||||
/**
|
||||
* The number of entities to insert.
|
||||
*/
|
||||
baseEntitiesCount: number;
|
||||
/**
|
||||
* The number of outgoing relations per entity.
|
||||
*/
|
||||
baseRelationsCount: number;
|
||||
/**
|
||||
* How "bunched up" relations are, as a number between 0 and 1.
|
||||
*
|
||||
* 0 means that relations are evenly distributed such that they form a uniform
|
||||
* mesh. 1 means that every single relation goes to one and the same exact
|
||||
* "victim" entity. Thus, the higher the number, the more unevenly distributed
|
||||
* the relations are.
|
||||
*/
|
||||
baseRelationsSkew: number;
|
||||
/**
|
||||
* The number of child entities emitted by each base entity.
|
||||
*/
|
||||
childrenCount: number;
|
||||
};
|
||||
|
||||
/**
|
||||
* Events that can occur during the load ingestion
|
||||
*/
|
||||
export type SyntheticLoadEvents = {
|
||||
onBeforeInsertBaseEntities?: () => void;
|
||||
onAfterInsertBaseEntities?: () => void;
|
||||
onError?: (error: Error) => void;
|
||||
};
|
||||
|
||||
/**
|
||||
* Throws if any of the options are invalid.
|
||||
*/
|
||||
function validateSyntheticLoadOptions(options: SyntheticLoadOptions): void {
|
||||
if (
|
||||
!Number.isSafeInteger(options.baseEntitiesCount) ||
|
||||
options.baseEntitiesCount <= 0
|
||||
) {
|
||||
throw new TypeError('baseEntitiesCount must be a nonnegative integer');
|
||||
} else if (
|
||||
!Number.isSafeInteger(options.baseRelationsCount) ||
|
||||
options.baseRelationsCount < 0
|
||||
) {
|
||||
throw new TypeError('baseRelationsCount must be a nonnegative integer');
|
||||
} else if (
|
||||
!Number.isFinite(options.baseRelationsSkew) ||
|
||||
options.baseRelationsSkew < 0 ||
|
||||
options.baseRelationsSkew > 1
|
||||
) {
|
||||
throw new TypeError('baseRelationsSkew must be a number between 0 and 1');
|
||||
} else if (
|
||||
!Number.isSafeInteger(options.childrenCount) ||
|
||||
options.childrenCount < 0
|
||||
) {
|
||||
throw new TypeError('childrenCount must be a nonnegative integer');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Some shared definitions
|
||||
*/
|
||||
export const common = {
|
||||
baseEntityName: (index: number) => `synthetic-${index}`,
|
||||
baseEntityNamespace: 'synthetic-load-base',
|
||||
baseEntity: (index: number): Entity => {
|
||||
return {
|
||||
apiVersion: 'backstage.io/v1alpha1',
|
||||
kind: 'Location',
|
||||
metadata: {
|
||||
name: common.baseEntityName(index),
|
||||
namespace: common.baseEntityNamespace,
|
||||
annotations: {
|
||||
'backstage.io/managed-by-location': `url:fake`,
|
||||
'backstage.io/managed-by-origin-location': `url:fake`,
|
||||
},
|
||||
},
|
||||
spec: {
|
||||
type: 'url',
|
||||
targets: [],
|
||||
},
|
||||
};
|
||||
},
|
||||
childEntityName: (baseEntity: Entity, index: number) =>
|
||||
`${baseEntity.metadata.name}-${index}`,
|
||||
childEntityNamespace: 'synthetic-load-child',
|
||||
childEntity: (baseEntity: Entity, index: number): Entity => {
|
||||
return {
|
||||
apiVersion: 'backstage.io/v1alpha1',
|
||||
kind: 'Location',
|
||||
metadata: {
|
||||
name: common.childEntityName(baseEntity, index),
|
||||
namespace: common.childEntityNamespace,
|
||||
annotations: {
|
||||
'backstage.io/managed-by-location': `url:fake`,
|
||||
'backstage.io/managed-by-origin-location': `url:fake`,
|
||||
},
|
||||
},
|
||||
spec: {
|
||||
type: 'url',
|
||||
targets: [],
|
||||
},
|
||||
};
|
||||
},
|
||||
kind: 'Location',
|
||||
relationType: 'loadTest',
|
||||
} as const;
|
||||
|
||||
/**
|
||||
* The entity provider that drives the initial base entity injection
|
||||
*/
|
||||
class SyntheticLoadEntitiesProvider implements EntityProvider {
|
||||
constructor(
|
||||
private readonly load: SyntheticLoadOptions,
|
||||
private readonly events: SyntheticLoadEvents,
|
||||
) {}
|
||||
|
||||
getProviderName(): string {
|
||||
return 'SyntheticLoadEntitiesProvider';
|
||||
}
|
||||
|
||||
async connect(connection: EntityProviderConnection): Promise<void> {
|
||||
// Defer this work so as to not block startup entirely
|
||||
setImmediate(async () => {
|
||||
try {
|
||||
this.events.onBeforeInsertBaseEntities?.();
|
||||
|
||||
const deferred: DeferredEntity[] = [];
|
||||
for (let index = 0; index < this.load.baseEntitiesCount; ++index) {
|
||||
deferred.push({ entity: common.baseEntity(index) });
|
||||
}
|
||||
|
||||
await connection.applyMutation({
|
||||
type: 'full',
|
||||
entities: deferred,
|
||||
});
|
||||
|
||||
this.events.onAfterInsertBaseEntities?.();
|
||||
} catch (error) {
|
||||
this.events.onError?.(error);
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Supporting processor for emitting children and relations
|
||||
*/
|
||||
class SyntheticLoadEntitiesProcessor implements CatalogProcessor {
|
||||
constructor(private readonly load: SyntheticLoadOptions) {}
|
||||
|
||||
getProcessorName(): string {
|
||||
return 'SyntheticLoadEntitiesProcessor';
|
||||
}
|
||||
|
||||
async postProcessEntity(
|
||||
entity: Entity,
|
||||
location: LocationSpec,
|
||||
emit: CatalogProcessorEmit,
|
||||
_cache: CatalogProcessorCache,
|
||||
): Promise<Entity> {
|
||||
const {
|
||||
baseEntitiesCount,
|
||||
baseRelationsCount,
|
||||
baseRelationsSkew,
|
||||
childrenCount,
|
||||
} = this.load;
|
||||
|
||||
if (entity.metadata.namespace === common.baseEntityNamespace) {
|
||||
for (let rc = 0; rc < baseRelationsCount; ++rc) {
|
||||
const relationIndex = Math.floor(
|
||||
Math.random() * baseEntitiesCount * (1 - baseRelationsSkew),
|
||||
);
|
||||
emit(
|
||||
processingResult.relation({
|
||||
source: {
|
||||
kind: common.kind,
|
||||
namespace: common.baseEntityNamespace,
|
||||
name: entity.metadata.name,
|
||||
},
|
||||
target: {
|
||||
kind: common.kind,
|
||||
namespace: common.baseEntityNamespace,
|
||||
name: common.baseEntityName(relationIndex),
|
||||
},
|
||||
type: common.relationType,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
for (let i = 0; i < childrenCount; ++i) {
|
||||
emit(processingResult.entity(location, common.childEntity(entity, i)));
|
||||
}
|
||||
}
|
||||
|
||||
return entity;
|
||||
}
|
||||
}
|
||||
|
||||
export const catalogModuleSyntheticLoadEntities = createBackendModule(
|
||||
(options: { load: SyntheticLoadOptions; events?: SyntheticLoadEvents }) => ({
|
||||
moduleId: 'syntheticLoadEntities',
|
||||
pluginId: 'catalog',
|
||||
register(reg) {
|
||||
reg.registerInit({
|
||||
deps: {
|
||||
catalog: catalogProcessingExtensionPoint,
|
||||
},
|
||||
async init({ catalog }) {
|
||||
const { load, events = {} } = options;
|
||||
|
||||
validateSyntheticLoadOptions(load);
|
||||
const provider = new SyntheticLoadEntitiesProvider(load, events);
|
||||
const processor = new SyntheticLoadEntitiesProcessor(load);
|
||||
|
||||
catalog.addEntityProvider(provider);
|
||||
catalog.addProcessor(processor);
|
||||
},
|
||||
});
|
||||
},
|
||||
}),
|
||||
);
|
||||
@@ -0,0 +1,22 @@
|
||||
/*
|
||||
* Copyright 2023 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
export const performanceTraceEnabled = !!process.env.PERFORMANCE_TRACE;
|
||||
|
||||
export const describePerformanceTest: jest.Describe = process.env
|
||||
.PERFORMANCE_TEST
|
||||
? describe
|
||||
: describe.skip;
|
||||
@@ -0,0 +1,195 @@
|
||||
/*
|
||||
* Copyright 2023 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import {
|
||||
coreServices,
|
||||
createServiceFactory,
|
||||
} from '@backstage/backend-plugin-api';
|
||||
import { TestDatabases, startTestBackend } from '@backstage/backend-test-utils';
|
||||
import { catalogPlugin } from '@backstage/plugin-catalog-backend/alpha';
|
||||
import { Knex } from 'knex';
|
||||
import { applyDatabaseMigrations } from '../../database/migrations';
|
||||
import {
|
||||
SyntheticLoadEvents,
|
||||
SyntheticLoadOptions,
|
||||
catalogModuleSyntheticLoadEntities,
|
||||
} from './lib/catalogModuleSyntheticLoadEntities';
|
||||
import { describePerformanceTest, performanceTraceEnabled } from './lib/env';
|
||||
|
||||
function defer<T>() {
|
||||
let resolve: (value: T | PromiseLike<T>) => void;
|
||||
let reject: (error?: unknown) => void;
|
||||
const promise = new Promise<T>((_resolve, _reject) => {
|
||||
resolve = _resolve;
|
||||
reject = _reject;
|
||||
});
|
||||
return { promise, resolve: resolve!, reject: reject! };
|
||||
}
|
||||
|
||||
const traceLog: typeof console.log = performanceTraceEnabled
|
||||
? console.log
|
||||
: () => {};
|
||||
|
||||
class Tracker {
|
||||
private insertBaseEntitiesStart: number | undefined;
|
||||
private insertBaseEntitiesEnd: number | undefined;
|
||||
private readonly deferred = defer<void>();
|
||||
|
||||
constructor(
|
||||
private readonly knex: Knex,
|
||||
private readonly load: SyntheticLoadOptions,
|
||||
) {}
|
||||
|
||||
events(): SyntheticLoadEvents {
|
||||
return {
|
||||
onBeforeInsertBaseEntities: () => {
|
||||
this.insertBaseEntitiesStart = Date.now();
|
||||
traceLog(`Inserting ${this.load.baseEntitiesCount} base entities`);
|
||||
},
|
||||
onAfterInsertBaseEntities: async () => {
|
||||
this.insertBaseEntitiesEnd = Date.now();
|
||||
|
||||
const insertDuration = (
|
||||
(this.insertBaseEntitiesEnd - this.insertBaseEntitiesStart!) /
|
||||
1000
|
||||
).toFixed(1);
|
||||
traceLog(
|
||||
`Inserted ${this.load.baseEntitiesCount} base entities in ${insertDuration} seconds`,
|
||||
);
|
||||
|
||||
await this.completionPolling();
|
||||
|
||||
const processingDuration = (
|
||||
(Date.now() - this.insertBaseEntitiesEnd) /
|
||||
1000
|
||||
).toFixed(1);
|
||||
traceLog(
|
||||
`Stitched ${this.load.baseEntitiesCount} entities in ${processingDuration} seconds`,
|
||||
);
|
||||
|
||||
this.deferred.resolve();
|
||||
},
|
||||
onError: error => {
|
||||
this.deferred.reject(error);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async completion(): Promise<void> {
|
||||
return this.deferred.promise;
|
||||
}
|
||||
|
||||
private completionPolling() {
|
||||
const { baseEntitiesCount, childrenCount } = this.load;
|
||||
const expectedTotal = baseEntitiesCount + baseEntitiesCount * childrenCount;
|
||||
|
||||
let processedTotal = 0;
|
||||
let stitchedTotal = 0;
|
||||
|
||||
return new Promise<void>((resolve, reject) => {
|
||||
const interval = setInterval(async () => {
|
||||
try {
|
||||
const processedCount = await this.knex('refresh_state')
|
||||
.count({ count: '*' })
|
||||
.whereNotNull('processed_entity')
|
||||
.then(rows => Number(rows[0].count));
|
||||
|
||||
const stitchedCount = await this.knex('final_entities')
|
||||
.count({ count: '*' })
|
||||
.whereNotNull('final_entity')
|
||||
.then(rows => Number(rows[0].count));
|
||||
|
||||
const processedDelta = processedCount - processedTotal;
|
||||
const processedPercent = (
|
||||
(processedCount / expectedTotal) *
|
||||
100
|
||||
).toFixed(1);
|
||||
const stitchedDelta = stitchedCount - stitchedTotal;
|
||||
const stitchedPercent = (
|
||||
(stitchedCount / expectedTotal) *
|
||||
100
|
||||
).toFixed(1);
|
||||
|
||||
const processedSummary = `${processedCount} (${processedPercent}%, ${processedDelta}/s)`;
|
||||
const stitchedSummary = `${stitchedCount} (${stitchedPercent}%, ${stitchedDelta}/s)`;
|
||||
traceLog(
|
||||
`Processed: ${processedSummary}\nStitched: ${stitchedSummary}`,
|
||||
);
|
||||
|
||||
processedTotal = processedCount;
|
||||
stitchedTotal = stitchedCount;
|
||||
|
||||
if (stitchedCount === expectedTotal) {
|
||||
clearInterval(interval);
|
||||
resolve();
|
||||
}
|
||||
} catch (error) {
|
||||
clearInterval(interval);
|
||||
reject(error);
|
||||
}
|
||||
}, 1000);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function staticDatabase(knex: Knex) {
|
||||
return createServiceFactory({
|
||||
service: coreServices.database,
|
||||
deps: {},
|
||||
createRootContext: () => undefined,
|
||||
factory: () => ({ getClient: async () => knex }),
|
||||
});
|
||||
}
|
||||
|
||||
jest.setTimeout(600_000);
|
||||
|
||||
describePerformanceTest('stitchingPerformance', () => {
|
||||
const databases = TestDatabases.create({
|
||||
ids: [/* 'MYSQL_8', */ 'POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'],
|
||||
});
|
||||
|
||||
it.each(databases.eachSupportedId())(
|
||||
'runs stitching in immediate mode, %p',
|
||||
async databaseId => {
|
||||
const knex = await databases.init(databaseId);
|
||||
await applyDatabaseMigrations(knex);
|
||||
|
||||
const load: SyntheticLoadOptions = {
|
||||
baseEntitiesCount: 1000,
|
||||
baseRelationsCount: 3,
|
||||
baseRelationsSkew: 0.3,
|
||||
childrenCount: 3,
|
||||
};
|
||||
|
||||
const tracker = new Tracker(knex, load);
|
||||
|
||||
const backend = await startTestBackend({
|
||||
services: [staticDatabase(knex)],
|
||||
features: [
|
||||
catalogPlugin(),
|
||||
catalogModuleSyntheticLoadEntities({
|
||||
load,
|
||||
events: tracker.events(),
|
||||
}),
|
||||
],
|
||||
});
|
||||
|
||||
await expect(tracker.completion()).resolves.toBeUndefined();
|
||||
await backend.stop();
|
||||
await knex.destroy();
|
||||
},
|
||||
);
|
||||
});
|
||||
Reference in New Issue
Block a user