diff --git a/.github/workflows/sync.yaml b/.github/workflows/sync.yaml
index fafdbb2..7903f01 100644
--- a/.github/workflows/sync.yaml
+++ b/.github/workflows/sync.yaml
@@ -22,9 +22,17 @@ jobs:
         run: pip install github-org-manager
       - name: Display information about github-org-manager
         run: gh-org-mgr --version
+      # Make a dry run in pull requests
+      - name: Synchronise settings with github-org-manager (pull request, dry)
+        if: github.event_name == 'pull_request'
+        run: gh-org-mgr sync -c . --dry
+        # Get app private key from secrets, set as environment variable
+        env:
+          GITHUB_APP_PRIVATE_KEY: ${{ secrets.GTHB_APP_PRIVATE_KEY }}
       # Only effectively sync on main branch
-      - name: Configure application, add login credentials
-        run: sed -i "s|__GITHUB_TOKEN__|${{ secrets.GTHB_TOKEN }}|" app.yaml
-      - name: Synchronise settings with github-org-manager
+      - name: Synchronise settings with github-org-manager (main branch, prod)
         if: github.ref == 'refs/heads/main'
         run: gh-org-mgr sync -c .
+        # Get app private key from secrets, set as environment variable
+        env:
+          GITHUB_APP_PRIVATE_KEY: ${{ secrets.GTHB_APP_PRIVATE_KEY }}
diff --git a/app.yaml b/app.yaml
index ceee029..6281b0c 100644
--- a/app.yaml
+++ b/app.yaml
@@ -4,4 +4,8 @@
 
 # Personal Oauth access token with required scopes
 # The current token, stored as repository secret, is owned by @mxmehl and will expire on 2025-05-02
-github_token: __GITHUB_TOKEN__
+# github_token:
+
+# GitHub App (if this is set, the personal access token (github_token) will be ignored)
+github_app_id: 1075380
+github_app_private_key: # set as environment variable from secrets