diff --git a/packages/create-app/bin/backstage-create-app b/packages/create-app/bin/backstage-create-app
index c9baf5d57c..2b445ca9af 100755
--- a/packages/create-app/bin/backstage-create-app
+++ b/packages/create-app/bin/backstage-create-app
@@ -21,7 +21,7 @@ const path = require('path');
 const isLocal = require('fs').existsSync(path.resolve(__dirname, '../src'));
 
 if (!isLocal || process.env.BACKSTAGE_E2E_CLI_TEST) {
-  require('../src');
+  require('..');
 } else {
   require('ts-node').register({
     transpileOnly: true,
diff --git a/packages/create-app/package.json b/packages/create-app/package.json
index 390c9cf2cb..282a1f692f 100644
--- a/packages/create-app/package.json
+++ b/packages/create-app/package.json
@@ -20,6 +20,9 @@
   "bin": {
     "backstage-create-app": "bin/backstage-create-app"
   },
+  "scripts": {
+    "build": "backstage-cli build --outputs cjs"
+  },
   "dependencies": {
     "commander": "^4.1.1",
     "chalk": "^4.0.0",
@@ -36,5 +39,10 @@
     "@types/recursive-readdir": "^2.2.0",
     "@types/ora": "^3.2.0",
     "ts-node": "^8.6.2"
-  }
+  },
+  "files": [
+    "bin",
+    "dist",
+    "templates"
+  ]
 }