add performance tests

Signed-off-by: Fredrik Adelöw <freben@gmail.com>

.changeset/purple-jokes-rhyme.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-catalog-backend': patch
+---
+
+Add a base plate for performance testing of the catalog

plugins/catalog-backend/src/tests/README.md

@@ -0,0 +1,3 @@
+# Overall Tests
+
+These are tests that do not apply to any one single class, and/or aren't regular unit tests.

plugins/catalog-backend/src/tests/integration.test.ts

@@ -14,45 +14,46 @@
  * limitations under the License.
  */
 
-import { Knex } from 'knex';
-import { Logger } from 'winston';
-import { ConfigReader } from '@backstage/config';
-import { JsonObject } from '@backstage/types';
-import { CatalogProcessingEngine, EntityProvider } from './index';
 import { DatabaseManager, getVoidLogger } from '@backstage/backend-common';
-import { PermissionEvaluator } from '@backstage/plugin-permission-common';
 import {
   Entity,
   EntityPolicies,
   stringifyEntityRef,
 } from '@backstage/catalog-model';
-import { defaultEntityDataParser } from './modules/util/parse';
-import { DefaultCatalogProcessingOrchestrator } from './processing/DefaultCatalogProcessingOrchestrator';
-import { applyDatabaseMigrations } from './database/migrations';
-import { DefaultCatalogDatabase } from './database/DefaultCatalogDatabase';
-import { DefaultProcessingDatabase } from './database/DefaultProcessingDatabase';
+import { ConfigReader } from '@backstage/config';
+import { InputError } from '@backstage/errors';
 import { ScmIntegrations } from '@backstage/integration';
-import { DefaultCatalogRulesEnforcer } from './ingestion/CatalogRules';
-import { Stitcher } from './stitching/Stitcher';
-import { DefaultEntitiesCatalog } from './service/DefaultEntitiesCatalog';
+import { LocationSpec } from '@backstage/plugin-catalog-common';
+import {
+  CatalogProcessorEmit,
+  EntityProvider,
+  EntityProviderConnection,
+  processingResult,
+} from '@backstage/plugin-catalog-node';
+import { PermissionEvaluator } from '@backstage/plugin-permission-common';
+import { JsonObject } from '@backstage/types';
+import { createHash } from 'crypto';
+import { Knex } from 'knex';
+import { Logger } from 'winston';
+import { EntitiesCatalog } from '../catalog/types';
+import { DefaultCatalogDatabase } from '../database/DefaultCatalogDatabase';
+import { DefaultProcessingDatabase } from '../database/DefaultProcessingDatabase';
+import { DefaultProviderDatabase } from '../database/DefaultProviderDatabase';
+import { applyDatabaseMigrations } from '../database/migrations';
+import { RefreshStateItem } from '../database/types';
+import { DefaultCatalogRulesEnforcer } from '../ingestion/CatalogRules';
+import { defaultEntityDataParser } from '../modules/util/parse';
 import {
   DefaultCatalogProcessingEngine,
   ProgressTracker,
-} from './processing/DefaultCatalogProcessingEngine';
-import { createHash } from 'crypto';
-import { DefaultRefreshService } from './service/DefaultRefreshService';
-import { connectEntityProviders } from './processing/connectEntityProviders';
-import { EntitiesCatalog } from './catalog/types';
-import { RefreshOptions, RefreshService } from './service/types';
-import {
-  CatalogProcessorEmit,
-  EntityProviderConnection,
-  LocationSpec,
-  processingResult,
-} from '@backstage/plugin-catalog-node';
-import { RefreshStateItem } from './database/types';
-import { DefaultProviderDatabase } from './database/DefaultProviderDatabase';
-import { InputError } from '@backstage/errors';
+} from '../processing/DefaultCatalogProcessingEngine';
+import { DefaultCatalogProcessingOrchestrator } from '../processing/DefaultCatalogProcessingOrchestrator';
+import { connectEntityProviders } from '../processing/connectEntityProviders';
+import { CatalogProcessingEngine } from '../processing/types';
+import { DefaultEntitiesCatalog } from '../service/DefaultEntitiesCatalog';
+import { DefaultRefreshService } from '../service/DefaultRefreshService';
+import { RefreshOptions, RefreshService } from '../service/types';
+import { Stitcher } from '../stitching/Stitcher';
 
 const voidLogger = getVoidLogger();
 

plugins/catalog-backend/src/tests/migrations.test.ts

@@ -18,7 +18,7 @@ import { Knex } from 'knex';
 import { TestDatabases } from '@backstage/backend-test-utils';
 import fs from 'fs';
 
-const migrationsDir = `${__dirname}/../migrations`;
+const migrationsDir = `${__dirname}/../../migrations`;
 const migrationsFiles = fs.readdirSync(migrationsDir).sort();
 
 async function migrateUpOnce(knex: Knex): Promise<void> {

plugins/catalog-backend/src/tests/performance/README.md

@@ -0,0 +1,6 @@
+# Catalog Performance Tests
+
+These are regular jest tests, but they are only enabled when the
+`PERFORMANCE_TEST` environment variable is set. You can also set the
+`PERFORMANCE_TRACE` environment variable to get additional trace output while
+the tests run.

plugins/catalog-backend/src/tests/performance/lib/catalogModuleSyntheticLoadEntities.ts

@@ -0,0 +1,257 @@
+/*
+ * Copyright 2023 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { createBackendModule } from '@backstage/backend-plugin-api';
+import { Entity } from '@backstage/catalog-model';
+import { LocationSpec } from '@backstage/plugin-catalog-common';
+import {
+  CatalogProcessor,
+  CatalogProcessorCache,
+  CatalogProcessorEmit,
+  DeferredEntity,
+  EntityProvider,
+  EntityProviderConnection,
+  processingResult,
+} from '@backstage/plugin-catalog-node';
+import { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node/alpha';
+
+/**
+ * Options for a fixed initial load of entities.
+ */
+export type SyntheticLoadOptions = {
+  /**
+   * The number of entities to insert.
+   */
+  baseEntitiesCount: number;
+  /**
+   * The number of outgoing relations per entity.
+   */
+  baseRelationsCount: number;
+  /**
+   * How "bunched up" relations are, as a number between 0 and 1.
+   *
+   * 0 means that relations are evenly distributed such that they form a uniform
+   * mesh. 1 means that every single relation goes to one and the same exact
+   * "victim" entity. Thus, the higher the number, the more unevenly distributed
+   * the relations are.
+   */
+  baseRelationsSkew: number;
+  /**
+   * The number of child entities emitted by each base entity.
+   */
+  childrenCount: number;
+};
+
+/**
+ * Events that can occur during the load ingestion
+ */
+export type SyntheticLoadEvents = {
+  onBeforeInsertBaseEntities?: () => void;
+  onAfterInsertBaseEntities?: () => void;
+  onError?: (error: Error) => void;
+};
+
+/**
+ * Throws if any of the options are invalid.
+ */
+function validateSyntheticLoadOptions(options: SyntheticLoadOptions): void {
+  if (
+    !Number.isSafeInteger(options.baseEntitiesCount) ||
+    options.baseEntitiesCount <= 0
+  ) {
+    throw new TypeError('baseEntitiesCount must be a nonnegative integer');
+  } else if (
+    !Number.isSafeInteger(options.baseRelationsCount) ||
+    options.baseRelationsCount < 0
+  ) {
+    throw new TypeError('baseRelationsCount must be a nonnegative integer');
+  } else if (
+    !Number.isFinite(options.baseRelationsSkew) ||
+    options.baseRelationsSkew < 0 ||
+    options.baseRelationsSkew > 1
+  ) {
+    throw new TypeError('baseRelationsSkew must be a number between 0 and 1');
+  } else if (
+    !Number.isSafeInteger(options.childrenCount) ||
+    options.childrenCount < 0
+  ) {
+    throw new TypeError('childrenCount must be a nonnegative integer');
+  }
+}
+
+/**
+ * Some shared definitions
+ */
+export const common = {
+  baseEntityName: (index: number) => `synthetic-${index}`,
+  baseEntityNamespace: 'synthetic-load-base',
+  baseEntity: (index: number): Entity => {
+    return {
+      apiVersion: 'backstage.io/v1alpha1',
+      kind: 'Location',
+      metadata: {
+        name: common.baseEntityName(index),
+        namespace: common.baseEntityNamespace,
+        annotations: {
+          'backstage.io/managed-by-location': `url:fake`,
+          'backstage.io/managed-by-origin-location': `url:fake`,
+        },
+      },
+      spec: {
+        type: 'url',
+        targets: [],
+      },
+    };
+  },
+  childEntityName: (baseEntity: Entity, index: number) =>
+    `${baseEntity.metadata.name}-${index}`,
+  childEntityNamespace: 'synthetic-load-child',
+  childEntity: (baseEntity: Entity, index: number): Entity => {
+    return {
+      apiVersion: 'backstage.io/v1alpha1',
+      kind: 'Location',
+      metadata: {
+        name: common.childEntityName(baseEntity, index),
+        namespace: common.childEntityNamespace,
+        annotations: {
+          'backstage.io/managed-by-location': `url:fake`,
+          'backstage.io/managed-by-origin-location': `url:fake`,
+        },
+      },
+      spec: {
+        type: 'url',
+        targets: [],
+      },
+    };
+  },
+  kind: 'Location',
+  relationType: 'loadTest',
+} as const;
+
+/**
+ * The entity provider that drives the initial base entity injection
+ */
+class SyntheticLoadEntitiesProvider implements EntityProvider {
+  constructor(
+    private readonly load: SyntheticLoadOptions,
+    private readonly events: SyntheticLoadEvents,
+  ) {}
+
+  getProviderName(): string {
+    return 'SyntheticLoadEntitiesProvider';
+  }
+
+  async connect(connection: EntityProviderConnection): Promise<void> {
+    // Defer this work so as to not block startup entirely
+    setImmediate(async () => {
+      try {
+        this.events.onBeforeInsertBaseEntities?.();
+
+        const deferred: DeferredEntity[] = [];
+        for (let index = 0; index < this.load.baseEntitiesCount; ++index) {
+          deferred.push({ entity: common.baseEntity(index) });
+        }
+
+        await connection.applyMutation({
+          type: 'full',
+          entities: deferred,
+        });
+
+        this.events.onAfterInsertBaseEntities?.();
+      } catch (error) {
+        this.events.onError?.(error);
+      }
+    });
+  }
+}
+
+/**
+ * Supporting processor for emitting children and relations
+ */
+class SyntheticLoadEntitiesProcessor implements CatalogProcessor {
+  constructor(private readonly load: SyntheticLoadOptions) {}
+
+  getProcessorName(): string {
+    return 'SyntheticLoadEntitiesProcessor';
+  }
+
+  async postProcessEntity(
+    entity: Entity,
+    location: LocationSpec,
+    emit: CatalogProcessorEmit,
+    _cache: CatalogProcessorCache,
+  ): Promise<Entity> {
+    const {
+      baseEntitiesCount,
+      baseRelationsCount,
+      baseRelationsSkew,
+      childrenCount,
+    } = this.load;
+
+    if (entity.metadata.namespace === common.baseEntityNamespace) {
+      for (let rc = 0; rc < baseRelationsCount; ++rc) {
+        const relationIndex = Math.floor(
+          Math.random() * baseEntitiesCount * (1 - baseRelationsSkew),
+        );
+        emit(
+          processingResult.relation({
+            source: {
+              kind: common.kind,
+              namespace: common.baseEntityNamespace,
+              name: entity.metadata.name,
+            },
+            target: {
+              kind: common.kind,
+              namespace: common.baseEntityNamespace,
+              name: common.baseEntityName(relationIndex),
+            },
+            type: common.relationType,
+          }),
+        );
+      }
+
+      for (let i = 0; i < childrenCount; ++i) {
+        emit(processingResult.entity(location, common.childEntity(entity, i)));
+      }
+    }
+
+    return entity;
+  }
+}
+
+export const catalogModuleSyntheticLoadEntities = createBackendModule(
+  (options: { load: SyntheticLoadOptions; events?: SyntheticLoadEvents }) => ({
+    moduleId: 'syntheticLoadEntities',
+    pluginId: 'catalog',
+    register(reg) {
+      reg.registerInit({
+        deps: {
+          catalog: catalogProcessingExtensionPoint,
+        },
+        async init({ catalog }) {
+          const { load, events = {} } = options;
+
+          validateSyntheticLoadOptions(load);
+          const provider = new SyntheticLoadEntitiesProvider(load, events);
+          const processor = new SyntheticLoadEntitiesProcessor(load);
+
+          catalog.addEntityProvider(provider);
+          catalog.addProcessor(processor);
+        },
+      });
+    },
+  }),
+);

plugins/catalog-backend/src/tests/performance/lib/env.ts

@@ -0,0 +1,22 @@
+/*
+ * Copyright 2023 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export const performanceTraceEnabled = !!process.env.PERFORMANCE_TRACE;
+
+export const describePerformanceTest: jest.Describe = process.env
+  .PERFORMANCE_TEST
+  ? describe
+  : describe.skip;

plugins/catalog-backend/src/tests/performance/stitchingPerformance.test.ts

@@ -0,0 +1,195 @@
+/*
+ * Copyright 2023 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {
+  coreServices,
+  createServiceFactory,
+} from '@backstage/backend-plugin-api';
+import { TestDatabases, startTestBackend } from '@backstage/backend-test-utils';
+import { catalogPlugin } from '@backstage/plugin-catalog-backend/alpha';
+import { Knex } from 'knex';
+import { applyDatabaseMigrations } from '../../database/migrations';
+import {
+  SyntheticLoadEvents,
+  SyntheticLoadOptions,
+  catalogModuleSyntheticLoadEntities,
+} from './lib/catalogModuleSyntheticLoadEntities';
+import { describePerformanceTest, performanceTraceEnabled } from './lib/env';
+
+function defer<T>() {
+  let resolve: (value: T | PromiseLike<T>) => void;
+  let reject: (error?: unknown) => void;
+  const promise = new Promise<T>((_resolve, _reject) => {
+    resolve = _resolve;
+    reject = _reject;
+  });
+  return { promise, resolve: resolve!, reject: reject! };
+}
+
+const traceLog: typeof console.log = performanceTraceEnabled
+  ? console.log
+  : () => {};
+
+class Tracker {
+  private insertBaseEntitiesStart: number | undefined;
+  private insertBaseEntitiesEnd: number | undefined;
+  private readonly deferred = defer<void>();
+
+  constructor(
+    private readonly knex: Knex,
+    private readonly load: SyntheticLoadOptions,
+  ) {}
+
+  events(): SyntheticLoadEvents {
+    return {
+      onBeforeInsertBaseEntities: () => {
+        this.insertBaseEntitiesStart = Date.now();
+        traceLog(`Inserting ${this.load.baseEntitiesCount} base entities`);
+      },
+      onAfterInsertBaseEntities: async () => {
+        this.insertBaseEntitiesEnd = Date.now();
+
+        const insertDuration = (
+          (this.insertBaseEntitiesEnd - this.insertBaseEntitiesStart!) /
+          1000
+        ).toFixed(1);
+        traceLog(
+          `Inserted ${this.load.baseEntitiesCount} base entities in ${insertDuration} seconds`,
+        );
+
+        await this.completionPolling();
+
+        const processingDuration = (
+          (Date.now() - this.insertBaseEntitiesEnd) /
+          1000
+        ).toFixed(1);
+        traceLog(
+          `Stitched ${this.load.baseEntitiesCount} entities in ${processingDuration} seconds`,
+        );
+
+        this.deferred.resolve();
+      },
+      onError: error => {
+        this.deferred.reject(error);
+      },
+    };
+  }
+
+  async completion(): Promise<void> {
+    return this.deferred.promise;
+  }
+
+  private completionPolling() {
+    const { baseEntitiesCount, childrenCount } = this.load;
+    const expectedTotal = baseEntitiesCount + baseEntitiesCount * childrenCount;
+
+    let processedTotal = 0;
+    let stitchedTotal = 0;
+
+    return new Promise<void>((resolve, reject) => {
+      const interval = setInterval(async () => {
+        try {
+          const processedCount = await this.knex('refresh_state')
+            .count({ count: '*' })
+            .whereNotNull('processed_entity')
+            .then(rows => Number(rows[0].count));
+
+          const stitchedCount = await this.knex('final_entities')
+            .count({ count: '*' })
+            .whereNotNull('final_entity')
+            .then(rows => Number(rows[0].count));
+
+          const processedDelta = processedCount - processedTotal;
+          const processedPercent = (
+            (processedCount / expectedTotal) *
+            100
+          ).toFixed(1);
+          const stitchedDelta = stitchedCount - stitchedTotal;
+          const stitchedPercent = (
+            (stitchedCount / expectedTotal) *
+            100
+          ).toFixed(1);
+
+          const processedSummary = `${processedCount} (${processedPercent}%, ${processedDelta}/s)`;
+          const stitchedSummary = `${stitchedCount} (${stitchedPercent}%, ${stitchedDelta}/s)`;
+          traceLog(
+            `Processed: ${processedSummary}\nStitched:  ${stitchedSummary}`,
+          );
+
+          processedTotal = processedCount;
+          stitchedTotal = stitchedCount;
+
+          if (stitchedCount === expectedTotal) {
+            clearInterval(interval);
+            resolve();
+          }
+        } catch (error) {
+          clearInterval(interval);
+          reject(error);
+        }
+      }, 1000);
+    });
+  }
+}
+
+function staticDatabase(knex: Knex) {
+  return createServiceFactory({
+    service: coreServices.database,
+    deps: {},
+    createRootContext: () => undefined,
+    factory: () => ({ getClient: async () => knex }),
+  });
+}
+
+jest.setTimeout(600_000);
+
+describePerformanceTest('stitchingPerformance', () => {
+  const databases = TestDatabases.create({
+    ids: [/* 'MYSQL_8', */ 'POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'],
+  });
+
+  it.each(databases.eachSupportedId())(
+    'runs stitching in immediate mode, %p',
+    async databaseId => {
+      const knex = await databases.init(databaseId);
+      await applyDatabaseMigrations(knex);
+
+      const load: SyntheticLoadOptions = {
+        baseEntitiesCount: 1000,
+        baseRelationsCount: 3,
+        baseRelationsSkew: 0.3,
+        childrenCount: 3,
+      };
+
+      const tracker = new Tracker(knex, load);
+
+      const backend = await startTestBackend({
+        services: [staticDatabase(knex)],
+        features: [
+          catalogPlugin(),
+          catalogModuleSyntheticLoadEntities({
+            load,
+            events: tracker.events(),
+          }),
+        ],
+      });
+
+      await expect(tracker.completion()).resolves.toBeUndefined();
+      await backend.stop();
+      await knex.destroy();
+    },
+  );
+});