diff --git a/.changeset/curly-rats-itch.md b/.changeset/curly-rats-itch.md
new file mode 100644
index 0000000000..b6fba9dc1e
--- /dev/null
+++ b/.changeset/curly-rats-itch.md
@@ -0,0 +1,5 @@
+---
+'@backstage/cli': patch
+---
+
+Tweak the Jest Caching loader to only operate when in `watch` mode
diff --git a/packages/cli/config/jest.js b/packages/cli/config/jest.js
index 8b6c37bda4..18f563a3a3 100644
--- a/packages/cli/config/jest.js
+++ b/packages/cli/config/jest.js
@@ -25,6 +25,11 @@ const envOptions = {
   enableSourceMaps: Boolean(process.env.ENABLE_SOURCE_MAPS),
 };
 
+if (envOptions.nextTests) {
+  // Needed so that, at import-time, it can hook into Jest's internals.
+  require('./jestCachingModuleLoader');
+}
+
 const transformIgnorePattern = [
   '@material-ui',
   'ajv',
diff --git a/packages/cli/config/jestCachingModuleLoader.js b/packages/cli/config/jestCachingModuleLoader.js
index 9ef29d7485..7858a7cfb7 100644
--- a/packages/cli/config/jestCachingModuleLoader.js
+++ b/packages/cli/config/jestCachingModuleLoader.js
@@ -21,6 +21,7 @@ const fileTransformCache = new Map();
 const scriptTransformCache = new Map();
 
 let runtimeGeneration = 0;
+let isWatchMode;
 
 module.exports = class CachingJestRuntime extends JestRuntime {
   // Each Jest run creates a new runtime, including when rerunning tests in
@@ -28,6 +29,10 @@ module.exports = class CachingJestRuntime extends JestRuntime {
   __runtimeGeneration = runtimeGeneration++;
 
   transformFile(filename, options) {
+    if (!isWatchMode) {
+      return super.transformFile(filename, options);
+    }
+
     const entry = fileTransformCache.get(filename);
     if (entry) {
       // Only check modification time if it's from a different runtime generation
@@ -79,3 +84,11 @@ module.exports = class CachingJestRuntime extends JestRuntime {
     return script;
   }
 };
+
+// Inject hook into createHasteMap, as it's the only way that we can
+// determine (from our scope here) if we're in "watch mode" or not.
+const originalCreateHasteMap = JestRuntime.createHasteMap;
+JestRuntime.createHasteMap = (config, options = undefined) => {
+  isWatchMode = options && options.watch;
+  return originalCreateHasteMap(config, options);
+};