diff --git a/.github/workflows/automate_changeset_feedback.yml b/.github/workflows/automate_changeset_feedback.yml
index 700eae7d0c..d27f0b84e0 100644
--- a/.github/workflows/automate_changeset_feedback.yml
+++ b/.github/workflows/automate_changeset_feedback.yml
@@ -26,70 +26,15 @@ jobs:
           # Fetch the commit that's merged into the base rather than the target ref
           # This will let us diff only the contents of the PR, without fetching more history
           ref: 'refs/pull/${{ github.event.pull_request.number }}/merge'
-
       - name: fetch base
         run: git fetch --depth 1 origin ${{ github.base_ref }}
-
-        # We avoid using the in-source script since this workflow has elevated permissions that we don't want to expose
-      - name: Generate Feedback
-        id: generate-feedback
-        run: |
-          rm -f generate.js
-          wget -O generate.js https://raw.githubusercontent.com/backstage/backstage/master/scripts/generate-changeset-feedback.js 1>&2
-          node generate.js FETCH_HEAD > feedback.txt
-
-      - name: Post Feedback
-        uses: actions/github-script@v6
-        env:
-          ISSUE_NUMBER: ${{ github.event.pull_request.number }}
+      - uses: backstage/actions/changeset-feedback@v0.5.12
+        name: Generate feedback
         with:
-          script: |
-            const owner = "backstage";
-            const repo = "backstage";
-            const marker = "<!-- changeset-feedback -->";
-            const feedback = require('fs').readFileSync('feedback.txt', 'utf8');
-            const issue_number = Number(process.env.ISSUE_NUMBER);
-            const body = feedback.trim() ? feedback + marker : undefined
-
-            const existingComments = await github.paginate(github.rest.issues.listComments, {
-              owner,
-              repo,
-              issue_number,
-            });
-
-            const existingComment = existingComments.find((c) =>
-              c.user.login === "github-actions[bot]" &&
-              c.body.includes(marker)
-            );
-
-            if (existingComment) {
-              if (body) {
-                if (existingComment.body !== body) {
-                  console.log(`updating existing comment in #${issue_number}`);
-                  await github.rest.issues.updateComment({
-                    owner,
-                    repo,
-                    comment_id: existingComment.id,
-                    body,
-                  });
-                } else {
-                  console.log(`skipped update of identical comment in #${issue_number}`);
-                }
-              } else {
-                console.log(`removing comment from #${issue_number}`);
-                await github.rest.issues.deleteComment({
-                  owner,
-                  repo,
-                  comment_id: existingComment.id,
-                  body,
-                });
-              }
-            } else if (body) {
-              console.log(`creating comment for #${issue_number}`);
-              await github.rest.issues.createComment({
-                owner,
-                repo,
-                issue_number,
-                body,
-              });
-            }
+          diffRef: 'origin/master'
+          marker: <!-- changeset-feedback -->
+          issue-number: ${{ github.event.pull_request.number }}
+          botUsername: github-actions[bot]
+          app-id: ${{ secrets.BACKSTAGE_GOALIE_APPLICATION_ID }}
+          private-key: ${{ secrets.BACKSTAGE_GOALIE_PRIVATE_KEY }}
+          installation-id: ${{ secrets.BACKSTAGE_GOALIE_INSTALLATION_ID }}