diff --git a/.github/workflows/cli.yml b/.github/workflows/cli.yml
index a0c100906d..0dd8036e84 100644
--- a/.github/workflows/cli.yml
+++ b/.github/workflows/cli.yml
@@ -65,7 +65,7 @@ jobs:
         working-directory: ${{ runner.temp }}/test-app
         env:
           BACKSTAGE_E2E_CLI_TEST: true
-      - name: run e2e tests in test app
+      - name: e2e test newly created app
         run: yarn test:e2e:ci
         working-directory: ${{ runner.temp }}/test-app/packages/app
         env:
diff --git a/packages/cli/templates/default-app/packages/app/package.json.hbs b/packages/cli/templates/default-app/packages/app/package.json.hbs
index 323c0963bd..2e607d6086 100644
--- a/packages/cli/templates/default-app/packages/app/package.json.hbs
+++ b/packages/cli/templates/default-app/packages/app/package.json.hbs
@@ -22,6 +22,7 @@
     "@types/jest": "^24.0.0",
     "@types/node": "^12.0.0",
     "@types/react-router-dom": "^5.1.3",
+    "cross-env": "^7.0.0",
     "cypress": "^4.2.0",
     "eslint-plugin-cypress": "^2.10.3",
     "start-server-and-test": "^1.10.11"
@@ -31,8 +32,8 @@
     "bundle": "backstage-cli app:build",
     "test": "backstage-cli test",
     "lint": "backstage-cli lint",
-    "test:e2e": "PORT=3001 start-server-and-test start http://localhost:3001 cy:dev",
-    "test:e2e:ci": "PORT=3001 start-server-and-test start http://localhost:3001 cy:run",
+    "test:e2e": "cross-env PORT=3001 start-server-and-test start http://localhost:3001 cy:dev",
+    "test:e2e:ci": "cross-env PORT=3001 start-server-and-test start http://localhost:3001 cy:run",
     "cy:dev": "cypress open",
     "cy:run": "cypress run"
   },