Notify slack of failing CI on master or release branches (#30244)

* notify slack of failing CI on master or release branches

* review comment updates

.github/workflows/rerun-workflows.yml

@@ -19,6 +19,10 @@ jobs:
           echo "View the failed run attempt (#${{ github.event.workflow_run.run_attempt }}) using the following link:" >> $GITHUB_STEP_SUMMARY
           echo "${{ github.event.workflow_run.html_url }}" >> $GITHUB_STEP_SUMMARY
 
+      - uses: actions/setup-node@v3
+        with:
+          node-version: '16'
+      - run: npm install @slack/web-api
       - name: Trigger a re-run
         uses: actions/github-script@v6
         with:
@@ -26,10 +30,67 @@ jobs:
             const MAX_ATTEMPTS = 2;
             const ATTEMPT = ${{ github.event.workflow_run.run_attempt }};
 
+            const author = `@${context.actor}`;
+            const breaking_commit = context.sha;
+            const branch = context.ref;
+
             if (ATTEMPT <= MAX_ATTEMPTS) {
               github.rest.actions.reRunWorkflowFailedJobs({
                 owner: context.repo.owner,
                 repo: context.repo.repo,
                 run_id: ${{ github.event.workflow_run.id }},
               });
+            } else {
+              // notify slack of repeated failure
+              const WebClient = require('@slack/web-api');
+              const slack = new WebClient(${{ secrets.SLACK_BOT_TOKEN }});
+
+              await slack.chat.postMessage({
+                channel: 'engineering-ci',
+                blocks: [
+                  {
+                    "type": "header",
+                    "text": {
+                      "type": "plain_text",
+                      "text": `:alert: CI is failing on ${branch} :alert:`,
+                      "emoji": true,
+                    }
+                  },
+                ],
+                attachments: [{
+                  color: '#f85149',
+                  blocks: [
+                    {
+                      "type": "section",
+                      "text": {
+                        "type": "mrkdwn",
+                        "text": `Commit <https://github.com/metabase/metabase/commit/${breaking_commit}|${breaking_commit.slice(0,7)}> by ${author} has failing tests on the \`${branch}\` branch . :sad-panda:\nPlease fix ASAP. Emoji this message when it's fixed.`
+                      }
+                    },
+                    {
+                      "type": "actions",
+                      "elements": [
+                        {
+                          "type": "button",
+                          "text": {
+                            "type": "plain_text",
+                            "text": "Failing Commit",
+                            "emoji": true
+                          },
+                          "url": `https://github.com/metabase/metabase/commit/${breaking_commit}`
+                        },
+                        {
+                          "type": "button",
+                          "text": {
+                            "type": "plain_text",
+                            "text": "Recent commits to master",
+                            "emoji": true
+                          },
+                          "url": `https://github.com/metabase/metabase/commits/${branch}`
+                        }
+                      ]
+                    }
+                  ]
+                }]
+              });
             }