diff --git a/.changeset/loud-kids-dance.md b/.changeset/loud-kids-dance.md
index 035e038f58..4fd32ac348 100644
--- a/.changeset/loud-kids-dance.md
+++ b/.changeset/loud-kids-dance.md
@@ -1,8 +1,5 @@
 ---
 '@backstage/cli': patch
-'@backstage/create-app': patch
 ---
 
-Append `-credentials.yaml` to credentials file generated by `backstage-cli create-github-app`.
-
-Add `*-credentials.yaml` to gitignore to prevent accidental commits.
+Append `-credentials.yaml` to credentials file generated by `backstage-cli create-github-app` and display warning about sensitive contents.
diff --git a/packages/cli/src/commands/create-github-app/index.ts b/packages/cli/src/commands/create-github-app/index.ts
index 2dd0d31232..cd9e8dbe09 100644
--- a/packages/cli/src/commands/create-github-app/index.ts
+++ b/packages/cli/src/commands/create-github-app/index.ts
@@ -30,5 +30,10 @@ export default async (org: string) => {
   const content = `# Name: ${name}\n${stringifyYaml(config)}`;
   await fs.writeFile(paths.resolveTargetRoot(fileName), content);
   console.log(`GitHub App configuration written to ${chalk.cyan(fileName)}`);
+  console.log(
+    chalk.yellow(
+      'This file contains sensitive credentials, it should not be committed to version control and handled with care!',
+    ),
+  );
   // TODO: log instructions on how to use the newly created app configuration.
 };