diff --git a/.changeset/loud-kids-dance.md b/.changeset/loud-kids-dance.md
new file mode 100644
index 0000000000..4fd32ac348
--- /dev/null
+++ b/.changeset/loud-kids-dance.md
@@ -0,0 +1,5 @@
+---
+'@backstage/cli': patch
+---
+
+Append `-credentials.yaml` to credentials file generated by `backstage-cli create-github-app` and display warning about sensitive contents.
diff --git a/.changeset/spoon-fork.md b/.changeset/spoon-fork.md
new file mode 100644
index 0000000000..5b8620f94e
--- /dev/null
+++ b/.changeset/spoon-fork.md
@@ -0,0 +1,12 @@
+---
+'@backstage/create-app': patch
+---
+
+Add `*-credentials.yaml` to gitignore to prevent accidental commits of sensitive credential information.
+
+To apply this change to an existing installation, add these lines to your `.gitignore`
+
+```gitignore
+# Sensitive credentials
+*-credentials.yaml
+```
diff --git a/.gitignore b/.gitignore
index 3334bf956d..57ad74c5cc 100644
--- a/.gitignore
+++ b/.gitignore
@@ -130,3 +130,6 @@ site
 
 # Local configuration files
 *.local.yaml
+
+# Sensitive credentials
+*-credentials.yaml
diff --git a/packages/cli/src/commands/create-github-app/index.ts b/packages/cli/src/commands/create-github-app/index.ts
index b8d232674e..cd9e8dbe09 100644
--- a/packages/cli/src/commands/create-github-app/index.ts
+++ b/packages/cli/src/commands/create-github-app/index.ts
@@ -26,9 +26,14 @@ import { GithubCreateAppServer } from './GithubCreateAppServer';
 export default async (org: string) => {
   const { slug, name, ...config } = await GithubCreateAppServer.run({ org });
 
-  const fileName = `github-app-${slug}.yaml`;
+  const fileName = `github-app-${slug}-credentials.yaml`;
   const content = `# Name: ${name}\n${stringifyYaml(config)}`;
   await fs.writeFile(paths.resolveTargetRoot(fileName), content);
   console.log(`GitHub App configuration written to ${chalk.cyan(fileName)}`);
+  console.log(
+    chalk.yellow(
+      'This file contains sensitive credentials, it should not be committed to version control and handled with care!',
+    ),
+  );
   // TODO: log instructions on how to use the newly created app configuration.
 };
diff --git a/packages/create-app/templates/default-app/.gitignore.hbs b/packages/create-app/templates/default-app/.gitignore.hbs
index 5f5cc739f4..4adebc5adc 100644
--- a/packages/create-app/templates/default-app/.gitignore.hbs
+++ b/packages/create-app/templates/default-app/.gitignore.hbs
@@ -30,4 +30,7 @@ dist-types
 site
 
 # Local configuration files
-*.local.yaml
\ No newline at end of file
+*.local.yaml
+
+# Sensitive credentials
+*-credentials.yaml