diff --git a/.github/workflows/automate_changeset_feedback.yml b/.github/workflows/automate_changeset_feedback.yml
index 0a95c6a7d9..fff6e95071 100644
--- a/.github/workflows/automate_changeset_feedback.yml
+++ b/.github/workflows/automate_changeset_feedback.yml
@@ -7,6 +7,7 @@ on:
 
 permissions:
   issues: write
+  pull-requests: write
 
 jobs:
   feedback:
@@ -15,17 +16,19 @@ jobs:
     steps:
       - uses: actions/checkout@v2
         with:
-          ref: ${{ github.head_ref }}
+          # Fetch the commit that's merged into the base rather than the target ref
+          # This will let us diff only the contents of the PR, without fetching more history
+          ref: 'refs/pull/${{ github.event.pull_request.number }}/merge'
 
       - name: fetch base
-        run: git fetch origin ${{ github.base_ref }}
+        run: git fetch --depth 1 origin ${{ github.base_ref }}
 
         # We avoid using the in-source script since this workflow has elevated permissions that we don't want to expose
       - name: Generate Feedback
         id: generate-feedback
         run: |
           wget -O generate.js https://raw.githubusercontent.com/backstage/backstage/master/scripts/generate-changeset-feedback.js 1>&2
-          node generate.js origin/${{ github.base_ref }} > feedback.txt
+          node generate.js FETCH_HEAD > feedback.txt
 
       - name: Post Feedback
         uses: actions/github-script@v5
@@ -33,7 +36,6 @@ jobs:
         env:
           ISSUE_NUMBER: ${{ github.event.pull_request.number }}
         with:
-          github-token: ${{ secrets.GH_SERVICE_ACCOUNT_TOKEN }}
           script: |
             const owner = "backstage";
             const repo = "backstage";
diff --git a/scripts/generate-changeset-feedback.js b/scripts/generate-changeset-feedback.js
index ca29e15530..588b13654c 100644
--- a/scripts/generate-changeset-feedback.js
+++ b/scripts/generate-changeset-feedback.js
@@ -80,35 +80,41 @@ async function listPackages() {
   }));
 }
 
-async function parseChangesets(filePaths) {
+async function loadChangesets(filePaths) {
   const changesets = [];
   for (const filePath of filePaths) {
     if (!filePath.startsWith('.changeset/') || !filePath.endsWith('.md')) {
       continue;
     }
-    const content = await fs.promises.readFile(filePath, 'utf8');
-    let lines = content.split(/\r?\n/);
+    try {
+      const content = await fs.promises.readFile(filePath, 'utf8');
+      let lines = content.split(/\r?\n/);
 
-    lines = lines.slice(lines.findIndex(line => line === '---') + 1);
-    lines = lines.slice(
-      0,
-      lines.findIndex(line => line === '---'),
-    );
+      lines = lines.slice(lines.findIndex(line => line === '---') + 1);
+      lines = lines.slice(
+        0,
+        lines.findIndex(line => line === '---'),
+      );
 
-    const bumps = new Map();
-    for (const line of lines) {
-      const match = line.match(/^'(.*)': (patch|minor|major)$/);
-      if (!match) {
-        throw new Error(`Invalid changeset line: ${line}`);
+      const bumps = new Map();
+      for (const line of lines) {
+        const match = line.match(/^'(.*)': (patch|minor|major)$/);
+        if (!match) {
+          throw new Error(`Invalid changeset line: ${line}`);
+        }
+
+        bumps.set(match[1], match[2]);
       }
 
-      bumps.set(match[1], match[2]);
+      changesets.push({
+        filePath,
+        bumps,
+      });
+    } catch (error) {
+      if (error.code !== 'ENOENT') {
+        throw error;
+      }
     }
-
-    changesets.push({
-      filePath,
-      bumps,
-    });
   }
 
   return changesets;
@@ -247,7 +253,7 @@ async function main() {
   const changedFiles = await listChangedFiles(diffRef);
   const packages = await listPackages();
 
-  const changesets = await parseChangesets(changedFiles);
+  const changesets = await loadChangesets(changedFiles);
   const changedPackages = await listChangedPackages(changedFiles, packages);
 
   process.stderr.write(