diff --git a/package.json b/package.json
index 129e50c95d..7752322535 100644
--- a/package.json
+++ b/package.json
@@ -26,7 +26,7 @@
     "backstage-create": "backstage-cli create --scope backstage --no-private",
     "create-plugin": "yarn backstage-create --select plugin",
     "remove-plugin": "backstage-cli remove-plugin",
-    "release": "node scripts/bump-create-app.js && node scripts/prepare-release.js && changeset version && yarn diff --yes && yarn prettier --write '{packages,plugins}/*/{package.json,CHANGELOG.md}' '.changeset/*.json' && yarn install",
+    "release": "node scripts/prepare-release.js && changeset version && yarn diff --yes && yarn prettier --write '{packages,plugins}/*/{package.json,CHANGELOG.md}' '.changeset/*.json' && yarn install",
     "prettier:check": "prettier --check .",
     "lerna": "lerna",
     "storybook": "yarn --cwd storybook start",
diff --git a/scripts/bump-create-app.js b/scripts/bump-create-app.js
deleted file mode 100755
index 2b552086af..0000000000
--- a/scripts/bump-create-app.js
+++ /dev/null
@@ -1,84 +0,0 @@
-#!/usr/bin/env node
-/*
- * Copyright 2020 The Backstage Authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/* eslint-disable import/no-extraneous-dependencies */
-
-const { resolve: resolvePath } = require('path');
-const fs = require('fs-extra');
-const { default: parseChangeset } = require('@changesets/parse');
-const { execFile: execFileCb } = require('child_process');
-const { promisify } = require('util');
-
-const exec = promisify(execFileCb);
-
-async function main() {
-  process.chdir(resolvePath(__dirname, '../.changeset'));
-
-  const fileNames = await fs.readdir('.');
-  const changesetNames = fileNames.filter(
-    name => name.endsWith('.md') && name !== 'README.md',
-  );
-
-  const changesets = await Promise.all(
-    changesetNames.map(async name => {
-      const content = await fs.readFile(name, 'utf8');
-      return { name, ...parseChangeset(content) };
-    }),
-  );
-
-  let excludeList = [];
-  if (await fs.pathExists('pre.json')) {
-    const data = await fs.readJSON('pre.json');
-    excludeList = data.changesets.map(name => `${name}.md`);
-  }
-  const hasCreateAppChanges = changesets
-    .filter(({ name }) => !excludeList.includes(name))
-    .map(changeset =>
-      changeset.releases.some(
-        release => release.name === '@backstage/create-app',
-      ),
-    )
-    .includes(true);
-
-  if (hasCreateAppChanges) {
-    console.log(
-      'Contains create-app changeset, no need to create additional changeset',
-    );
-    return;
-  }
-  const ts = Math.round(new Date().getTime() / 1000);
-  const fileName = `create-app-${ts}.md`;
-  console.log(`Creating ${fileName}`);
-  const data = `---
-'@backstage/create-app': patch
----\n
-Bumped create-app version.\n`;
-  await fs.writeFile(fileName, data);
-  await exec('git', ['config', 'user.name', `"github-actions[bot]"`]);
-  await exec('git', [
-    'config',
-    'user.email',
-    `"github-actions[bot]@users.noreply.github.com"`,
-  ]);
-  await exec('git', ['add', fileName]);
-  await exec('git', ['commit', '-m', 'Add create-app changeset']);
-}
-
-main().catch(error => {
-  console.error(error.stack);
-  process.exit(1);
-});
diff --git a/scripts/prepare-release.js b/scripts/prepare-release.js
index d5ad1842d4..a9b161a33a 100755
--- a/scripts/prepare-release.js
+++ b/scripts/prepare-release.js
@@ -22,6 +22,7 @@ const { getPackages } = require('@manypkg/get-packages');
 const path = require('path');
 const { execFile: execFileCb } = require('child_process');
 const { promisify } = require('util');
+const { default: parseChangeset } = require('@changesets/parse');
 
 const execFile = promisify(execFileCb);
 
@@ -332,6 +333,55 @@ async function updateBackstageReleaseVersion(repo, type) {
   );
 }
 
+/**
+ * Ensures that the changesets include a version bump of create-app otherwise
+ * generates a new patch changeset for create-app.
+ */
+async function ensureCreateAppChangeset() {
+  process.chdir(path.resolve(__dirname, '../.changeset'));
+
+  const fileNames = await fs.readdir('.');
+  const changesetNames = fileNames.filter(
+    name => name.endsWith('.md') && name !== 'README.md',
+  );
+
+  const changesets = await Promise.all(
+    changesetNames.map(async name => {
+      const content = await fs.readFile(name, 'utf8');
+      return { name, ...parseChangeset(content) };
+    }),
+  );
+
+  let excludeList = [];
+  if (await fs.pathExists('pre.json')) {
+    const data = await fs.readJSON('pre.json');
+    excludeList = data.changesets.map(name => `${name}.md`);
+  }
+  const hasCreateAppChanges = changesets
+    .filter(({ name }) => !excludeList.includes(name))
+    .map(changeset =>
+      changeset.releases.some(
+        release => release.name === '@backstage/create-app',
+      ),
+    )
+    .includes(true);
+
+  if (hasCreateAppChanges) {
+    console.log(
+      'Contains create-app changeset, no need to create additional changeset',
+    );
+    return;
+  }
+  const ts = Math.round(new Date().getTime() / 1000);
+  const fileName = `create-app-${ts}.md`;
+  console.log(`Creating ${fileName}`);
+  const data = `---
+'@backstage/create-app': patch
+---\n
+Bumped create-app version.\n`;
+  await fs.writeFile(fileName, data);
+}
+
 async function main() {
   const repo = await getPackages(__dirname);
   const branchName = await getCurrentBranch(repo);
@@ -344,6 +394,7 @@ async function main() {
   }
 
   await updateBackstageReleaseVersion(repo, isMainBranch ? 'minor' : 'patch');
+  await ensureCreateAppChangeset();
 }
 
 main().catch(error => {