cli: added experimental test configuration and module loader

Signed-off-by: Patrik Oldsberg <poldsberg@gmail.com>

.changeset/fast-cars-rest.md

@@ -0,0 +1,7 @@
+---
+'@backstage/cli': patch
+---
+
+Introduced a new experimental test configuration with a number of changes. It switches the coverage provider from `v8` to the default Babel provider, along with always enabling source maps in the Sucrase transform. It also adds a custom module loader that caches both file transforms and VM script objects across all projects in a test run, which provides a big performance boost when running tests from the project root, increasing speed and reducing memory usage.
+
+This new configuration is not enabled by default. It is enabled by setting the environment variable `BACKSTAGE_NEXT_TESTS` to a non-empty value.

.github/workflows/ci.yml

@@ -155,6 +155,7 @@ jobs:
         if: ${{ steps.yarn-lock.outcome == 'success' }}
         run: yarn lerna -- run test --since origin/master -- --coverage --runInBand
         env:
+          BACKSTAGE_NEXT_TESTS: 1
           BACKSTAGE_TEST_DISABLE_DOCKER: 1
           BACKSTAGE_TEST_DATABASE_POSTGRES13_CONNECTION_STRING: postgresql://postgres:postgres@localhost:${{ job.services.postgres13.ports[5432] }}
           BACKSTAGE_TEST_DATABASE_POSTGRES9_CONNECTION_STRING: postgresql://postgres:postgres@localhost:${{ job.services.postgres9.ports[5432] }}
@@ -166,6 +167,7 @@ jobs:
           yarn lerna -- run test -- --coverage --runInBand
           bash <(curl -s https://codecov.io/bash) -N $(git rev-parse FETCH_HEAD)
         env:
+          BACKSTAGE_NEXT_TESTS: 1
           BACKSTAGE_TEST_DISABLE_DOCKER: 1
           BACKSTAGE_TEST_DATABASE_POSTGRES13_CONNECTION_STRING: postgresql://postgres:postgres@localhost:${{ job.services.postgres13.ports[5432] }}
           BACKSTAGE_TEST_DATABASE_POSTGRES9_CONNECTION_STRING: postgresql://postgres:postgres@localhost:${{ job.services.postgres9.ports[5432] }}

.github/workflows/deploy_packages.yml

@@ -122,6 +122,7 @@ jobs:
           bash <(curl -s https://codecov.io/bash) -f packages/core-components/coverage/* -F core-components
           bash <(curl -s https://codecov.io/bash) -f packages/core-plugin-api/coverage/* -F core-plugin-api
         env:
+          BACKSTAGE_NEXT_TESTS: 1
           BACKSTAGE_TEST_DISABLE_DOCKER: 1
           BACKSTAGE_TEST_DATABASE_POSTGRES13_CONNECTION_STRING: postgresql://postgres:postgres@localhost:${{ job.services.postgres13.ports[5432] }}
           BACKSTAGE_TEST_DATABASE_POSTGRES9_CONNECTION_STRING: postgresql://postgres:postgres@localhost:${{ job.services.postgres9.ports[5432] }}

.github/workflows/verify_windows.yml

@@ -58,6 +58,7 @@ jobs:
       - name: test
         run: yarn lerna -- run test
         env:
+          BACKSTAGE_NEXT_TESTS: 1
           BACKSTAGE_TEST_DISABLE_DOCKER: 1
 
       # credit: https://github.com/appleboy/discord-action/issues/3#issuecomment-731426861

packages/cli/config/jest.js

@@ -20,6 +20,11 @@ const crypto = require('crypto');
 const glob = require('util').promisify(require('glob'));
 const { version } = require('../package.json');
 
+const envOptions = {
+  nextTests: Boolean(process.env.BACKSTAGE_NEXT_TESTS),
+  enableSourceMaps: Boolean(process.env.ENABLE_SOURCE_MAPS),
+};
+
 const transformIgnorePattern = [
   '@material-ui',
   '@rjsf',
@@ -118,7 +123,7 @@ async function getProjectConfig(targetPath, displayName) {
     ...(displayName && { displayName }),
     rootDir: path.resolve(targetPath, 'src'),
     coverageDirectory: path.resolve(targetPath, 'coverage'),
-    coverageProvider: 'v8',
+    coverageProvider: envOptions.nextTests ? undefined : 'v8',
     collectCoverageFrom: ['**/*.{js,jsx,ts,tsx,mjs,cjs}', '!**/*.d.ts'],
     moduleNameMapper: {
       '\\.(css|less|scss|sss|styl)$': require.resolve('jest-css-modules'),
@@ -127,7 +132,9 @@ async function getProjectConfig(targetPath, displayName) {
     transform: {
       '\\.(js|jsx|ts|tsx|mjs|cjs)$': [
         require.resolve('./jestSucraseTransform.js'),
-        { enableSourceMaps: Boolean(process.env.ENABLE_SOURCE_MAPS) },
+        {
+          enableSourceMaps: envOptions.enableSourceMaps || envOptions.nextTests,
+        },
       ],
       '\\.(bmp|gif|jpg|jpeg|png|frag|xml|svg|eot|woff|woff2|ttf)$':
         require.resolve('./jestFileTransform.js'),
@@ -137,6 +144,10 @@ async function getProjectConfig(targetPath, displayName) {
     // A bit more opinionated
     testMatch: ['**/*.test.{js,jsx,ts,tsx,mjs,cjs}'],
 
+    moduleLoader: envOptions.nextTests
+      ? require.resolve('./jestCachingModuleLoader')
+      : undefined,
+
     transformIgnorePatterns: [`/node_modules/(?:${transformIgnorePattern})/`],
 
     ...getRoleConfig(closestPkgJson?.backstage?.role),

packages/cli/config/jestCachingModuleLoader.js

@@ -0,0 +1,81 @@
+/*
+ * Copyright 2022 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+const fs = require('fs');
+const { default: JestRuntime } = require('jest-runtime');
+
+const fileTransformCache = new Map();
+const scriptTransformCache = new Map();
+
+let runtimeGeneration = 0;
+
+module.exports = class CachingJestRuntime extends JestRuntime {
+  // Each Jest run creates a new runtime, including when rerunning tests in
+  // watch mode. This keeps track of whether we've switched runtime instance.
+  __runtimeGeneration = runtimeGeneration++;
+
+  transformFile(filename, options) {
+    const entry = fileTransformCache.get(filename);
+    if (entry) {
+      // Only check modification time if it's from a different runtime generation
+      if (entry.generation === this.__runtimeGeneration) {
+        return entry.code;
+      }
+
+      // Keep track of the modification time of files so that we can properly
+      // reprocess them in watch mode.
+      const { mtimeMs } = fs.statSync(filename);
+      if (mtimeMs > entry.mtimeMs) {
+        const code = super.transformFile(filename, options);
+        fileTransformCache.set(filename, {
+          code,
+          mtimeMs,
+          generation: this.__runtimeGeneration,
+        });
+        return code;
+      }
+
+      fileTransformCache.set(filename, {
+        ...entry,
+        generation: this.__runtimeGeneration,
+      });
+      return entry.code;
+    }
+
+    const code = super.transformFile(filename, options);
+    fileTransformCache.set(filename, {
+      code,
+      mtimeMs: fs.statSync(filename).mtimeMs,
+      generation: this.__runtimeGeneration,
+    });
+    return code;
+  }
+
+  // This may or may not be a good idea. Theoretically I don't know why this would impact
+  // test correctness and flakiness, but it seems like it may introduce flakiness and strange failures.
+  // It does seem to speed up test execution by a fair amount though.
+  createScriptFromCode(scriptSource, filename) {
+    let script = scriptTransformCache.get(scriptSource);
+    if (!script) {
+      script = super.createScriptFromCode(scriptSource, filename);
+      // Tried to store the script object in a WeakRef here. It starts out at
+      // about 90% hit rate, but eventually drops all the way to 20%, and overall
+      // it seemed to increase memory usage by 20% or so.
+      scriptTransformCache.set(scriptSource, script);
+    }
+    return script;
+  }
+};

packages/cli/package.json

@@ -86,6 +86,7 @@
     "html-webpack-plugin": "^5.3.1",
     "inquirer": "^8.2.0",
     "jest": "^27.5.1",
+    "jest-runtime": "^27.5.1",
     "jest-css-modules": "^2.1.0",
     "jest-transform-yaml": "^1.0.0",
     "json-schema": "^0.4.0",

packages/cli/src/lib/parallel.ts

@@ -213,6 +213,7 @@ export async function runWorkerQueueThreads<TItem, TResult, TData>(
   return results;
 }
 
+/* istanbul ignore next */
 function workerQueueThread(
   workerFuncFactory: (
     data: unknown,
@@ -313,6 +314,7 @@ export async function runWorkerThreads<TResult, TData, TMessage>(
   );
 }
 
+/* istanbul ignore next */
 function workerThread(
   workerFunc: (
     data: unknown,