diff --git a/.changeset/odd-humans-exercise.md b/.changeset/odd-humans-exercise.md
new file mode 100644
index 0000000000..136ec86b69
--- /dev/null
+++ b/.changeset/odd-humans-exercise.md
@@ -0,0 +1,5 @@
+---
+'@backstage/cli': patch
+---
+
+Make the `create-github-app` command disable webhooks by default.
diff --git a/docs/plugins/github-apps.md b/docs/plugins/github-apps.md
index 87d23b45d5..51d3fb9854 100644
--- a/docs/plugins/github-apps.md
+++ b/docs/plugins/github-apps.md
@@ -43,6 +43,10 @@ root of the project which you can then use as an `include` in your
 `app-config.yaml`. You can go ahead and
 [skip ahead](#including-in-integrations-config) if you've already got an app.
 
+Note that the created app will have a webhook that is disabled by default and
+points to `smee.io`, which is intended for local development. There's also
+currently no part of Backstage that makes use of the webhook.
+
 ### GitHub Enterprise
 
 You have to create the GitHub Application manually using these
diff --git a/packages/cli/src/commands/create-github-app/GithubCreateAppServer.ts b/packages/cli/src/commands/create-github-app/GithubCreateAppServer.ts
index 45671c2ead..0ffc1a08ff 100644
--- a/packages/cli/src/commands/create-github-app/GithubCreateAppServer.ts
+++ b/packages/cli/src/commands/create-github-app/GithubCreateAppServer.ts
@@ -120,6 +120,7 @@ export class GithubCreateAppServer {
       redirect_url: `${baseUrl}/callback`,
       hook_attributes: {
         url: this.webhookUrl,
+        active: false,
       },
     };
     const manifestJson = JSON.stringify(manifest).replace(/\"/g, '"');