diff --git a/.github/workflows/rerun-workflows.yml b/.github/workflows/rerun-workflows.yml
index d93ba230cb244d6c8f7408931c64e3b948d172e8..fb5ac600ed5e24de310fb909ced4726f6931c68d 100644
--- a/.github/workflows/rerun-workflows.yml
+++ b/.github/workflows/rerun-workflows.yml
@@ -27,7 +27,7 @@ jobs:
       - name: Trigger a re-run
         uses: actions/github-script@v6
         with:
-          script: |
+          script: | # js
             const MAX_ATTEMPTS = 2;
             const ATTEMPT = "${{ github.event.workflow_run.run_attempt }}";
             const BRANCH = "${{ github.event.workflow_run.head_branch }}";
@@ -49,58 +49,27 @@ jobs:
 
               await slack.chat.postMessage({
                 channel: 'engineering-ci',
+                text: 'Failing tests',
                 blocks: [
                   {
                     "type": "header",
                     "text": {
                       "type": "plain_text",
-                      "text": `:alert: CI is failing on ${BRANCH} :alert:`,
+                      "text": `:warning: CI is failing on ${BRANCH}`,
                       "emoji": true,
                     }
                   },
                 ],
                 attachments: [{
-                  color: '#f85149',
+                  color: BRANCH === 'master' ? '#f85149' : "#ffce33",
                   blocks: [
                     {
                       "type": "section",
                       "text": {
                         "type": "mrkdwn",
-                        "text": `Commit <https://github.com/metabase/metabase/commit/${BREAKING_COMMIT}|${BREAKING_COMMIT.slice(0,7)}> by ${AUTHOR} has failing <${FAILED_RUN_URL}|${FAILED_RUN_NAME}> tests on the \`${BRANCH}\` branch . :sad-panda:\nPlease fix ASAP. \n\nEmoji this message when it's fixed.`
+                        "text": `Commit <https://github.com/metabase/metabase/commit/${BREAKING_COMMIT}|${BREAKING_COMMIT.slice(0,7)}> by ${AUTHOR} has failing <${FAILED_RUN_URL}|${FAILED_RUN_NAME}> tests on the <https://github.com/metabase/metabase/commits/${BRANCH}|\`${BRANCH}\`> branch. :sad-panda:`
                       }
                     },
-                    {
-                      "type": "actions",
-                      "elements": [
-                        {
-                          "type": "button",
-                          "text": {
-                            "type": "plain_text",
-                            "text": "Failing Commit",
-                            "emoji": true
-                          },
-                          "url": `https://github.com/metabase/metabase/commit/${BREAKING_COMMIT}`
-                        },
-                        {
-                          "type": "button",
-                          "text": {
-                            "type": "plain_text",
-                            "text": `Recent commits to ${BRANCH}`,
-                            "emoji": true
-                          },
-                          "url": `https://github.com/metabase/metabase/commits/${BRANCH}`
-                        },
-                        {
-                          "type": "button",
-                          "text": {
-                            "type": "plain_text",
-                            "text": "Failed CI Run",
-                            "emoji": true
-                          },
-                          "url": `${FAILED_RUN_URL}`
-                        }
-                      ]
-                    }
                   ]
                 }]
               });