diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml
index 92bd759e36..746b6c80de 100644
--- a/.github/workflows/pr.yaml
+++ b/.github/workflows/pr.yaml
@@ -1,14 +1,31 @@
 name: PR
 on:
   pull_request_target:
+    types:
+      - opened
+      - synchronize
+      - reopened
+      - closed
+  issue_comment:
+    types:
+      - created
+  pull_request_review_comment:
+    types:
+      - created
 
 jobs:
   sync:
     runs-on: ubuntu-latest
 
-    if: github.repository == 'backstage/backstage'
+    # Avoid running on issue comments
+    if: github.repository == 'backstage/backstage' && ( github.event.pull_request || github.event.issue.pull_request )
     steps:
       - name: PR sync
-        uses: backstage/actions/pr-sync@v0.2.2
+        uses: backstage/actions/pr-sync@v0.4.0
         with:
-          github-token: ${{ secrets.GH_SERVICE_ACCOUNT_TOKEN }}
+          app-id: ${{ secrets.BACKSTAGE_GOALIE_APPLICATION_ID }}
+          private-key: ${{ secrets.BACKSTAGE_GOALIE_PRIVATE_KEY }}
+          installation-id: ${{ secrets.BACKSTAGE_GOALIE_INSTALLATION_ID }}
+          project-id: PVT_kwDOBFKqdc02LQ
+          excluded-users: ${{ secrets.OOO_USERS }}
+          owning-teams: '@backstage/techdocs'