Move E2E download uberjar logic into a composite action (#34647)

* Move E2E download uberjar logic into a composite action

* Check out only the last 20 commits

* Always use the correct, last commit in a branch

This helps us avoid the slippery slope in which GitHub creates a new
merge commit, that doesn't exist in the repo, under our feet.

* Adapt the stress-test workflow to use new download action

* Increase the number of commits to search through

* Update .github/actions/e2e-download-uberjar/action.yml

Co-authored-by: Ryan Laurie <30528226+iethree@users.noreply.github.com>

* Log full artifact name

* Correctly download artifact from the current build

---------

Co-authored-by: Ryan Laurie <30528226+iethree@users.noreply.github.com>

.github/actions/e2e-download-uberjar/action.yml

+name: Download uberjar for E2E tests
+description: Download either a freshly built artifact or the one built in one of the previous commits.
+inputs:
+  edition:
+    description: Metabase edition.
+    required: true
+  was-built:
+    description: Was the artifact built during the current workflof run?
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+    - uses: actions/download-artifact@v3
+      if: ${{ inputs.was-built == 'true' }}
+      name: Retrieve uberjar artifact for ${{ inputs.edition }}
+      with:
+        name: metabase-${{ inputs.edition }}-${{ github.event.pull_request.head.sha || github.sha }}-uberjar
+
+    - name: Download previously stored uberjar
+      if: ${{ inputs.was-built == 'false' }}
+      uses: actions/github-script@v6
+      with:
+        script: | # js
+          const fs = require('fs');
+          const execSync = require("child_process").execSync;
+
+          const getCommit = (ref = "HEAD") => {
+            const [commit] = execSync(`git rev-parse ${ref}`, { encoding: "utf8" }).split(
+              "\n",
+            );
+            return commit;
+          };
+
+          const baseConfig = {
+            owner: context.repo.owner,
+            repo: context.repo.repo,
+          };
+
+          async function getArtifact(commit, depth = 0) {
+            if (depth > 20) {
+              throw new Error("Couldn't find the artifact!");
+            }
+
+            const artifactName = `metabase-${{ inputs.edition }}-${commit}-uberjar`;
+
+            const {
+              data: { artifacts },
+            } = await github.rest.actions.listArtifactsForRepo({
+              ...baseConfig,
+              name: artifactName,
+              per_page: 1,
+            });
+
+            const [artifact] = artifacts;
+
+            if (!artifact) {
+              const parentCommit = getCommit(`${commit}^`);
+              console.log(`Didn't find ${artifactName}.\n`);
+              console.log(`Switching to the parent commit: ${parentCommit}`);
+              return getArtifact(parentCommit, depth + 1);
+            } else {
+              return artifact;
+            }
+          }
+
+          const currentCommit = "${{ github.event.pull_request.head.sha || github.sha }}";
+          const { id } = await getArtifact(currentCommit);
+
+          const download = await github.rest.actions.downloadArtifact({
+            ...baseConfig,
+            artifact_id: id,
+            archive_format: 'zip',
+          });
+
+          fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/mb.zip`, Buffer.from(download.data));
+
+    - name: Unzip Metabase artifact containing an uberjar
+      if: ${{ inputs.was-built == 'false' }}
+      run: unzip mb.zip
+      shell: bash
+
+    - name: Get the version info
+      run: |
+        jar xf target/uberjar/metabase.jar version.properties
+        cat version.properties
+        mv version.properties resources/
+      shell: bash

.github/workflows/e2e-stress-test-flake-fix.yml

@@ -3,10 +3,6 @@ name: E2E Stress Test Flake Fix
 on:
   workflow_dispatch:
     inputs:
-      artifact:
-        description: 'Artifact ID'
-        type: string
-        required: true
       spec:
         description: 'Relative path of the target spec'
         type: string
@@ -32,6 +28,9 @@ jobs:
       TZ: US/Pacific # to make node match the instance tz
     steps:
       - uses: actions/checkout@v3
+        with:
+          # Important because we need previous commits hashes to find and download the uberjar!
+          fetch-depth: 20
 
       - name: Prepare Docker containers
         uses: ./.github/actions/e2e-prepare-containers
@@ -44,19 +43,11 @@ jobs:
           mysql: true
           mongo: true
 
-      - name: Download Metabase uberjar from a previously stored artifact
-        run: |
-          curl -L \
-            -H "Accept: application/vnd.github+json" \
-            -H "Authorization: Bearer $GITHUB_TOKEN" \
-            -H "X-GitHub-Api-Version: 2022-11-28" \
-            https://api.github.com/repos/metabase/metabase/actions/artifacts/$ARTIFACT_ID/zip \
-            -o mb.zip
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          ARTIFACT_ID: ${{ github.event.inputs.artifact }}
-      - name: Unzip Metabase artifact containing an uberjar
-        run: unzip mb.zip
+      - name: Download Metabase ${{ matrix.edition }} uberjar
+        uses: ./.github/actions/e2e-download-uberjar
+        with:
+          edition: 'ee'
+
       - name: Prepare front-end environment
         uses: ./.github/actions/prepare-frontend
       - name: Prepare JDK 11
@@ -69,10 +60,6 @@ jobs:
         uses: ./.github/actions/prepare-cypress
       - name: Run Snowplow micro
         uses: ./.github/actions/run-snowplow-micro
-      - name: Get Metabase version info
-        run: |
-          jar xf target/uberjar/metabase.jar version.properties
-          mv version.properties resources/
 
       - name: Stress-test ${{ github.event.inputs.spec }} ${{ github.event.inputs.burn_in }} times
         run: |

.github/workflows/e2e-tests.yml

@@ -50,81 +50,12 @@ jobs:
         id: e2e-matrix
         uses: ./.github/actions/build-e2e-matrix
 
-  download_uberjar:
-    runs-on: ubuntu-22.04
-    timeout-minutes: 10
-    needs: [files-changed, e2e-matrix-builder]
-    if: |
-      !cancelled() &&
-      github.event.pull_request.draft == false &&
-      needs.e2e-matrix-builder.result == 'success' &&
-      needs.files-changed.outputs.e2e_specs == 'true' &&
-      needs.files-changed.outputs.e2e_all != 'true'
-    strategy:
-      matrix:
-        edition: [oss, ee]
-    outputs:
-      enterprise_download_link: ${{ steps.zip_url.outputs.ee_dl }}
-      oss_download_link: ${{ steps.zip_url.outputs.oss_dl }}
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-      - name: Artifact zip URL
-        id: zip_url
-        run: |
-          current_commit="HEAD"
-
-          get_parent () {
-            parent_commit=$(git rev-parse $1^)
-            echo $parent_commit
-          }
-
-          get_download_link () {
-            parent_commit=$(get_parent $1)
-            current_page=${2:-1}
-
-            artifacts=$(curl -sL \
-            -H "Accept: application/vnd.github+json" \
-            -H "Authorization: Bearer $GITHUB_TOKEN" \
-            -H "X-GitHub-Api-Version: 2022-11-28" \
-            "https://api.github.com/repos/metabase/metabase/actions/artifacts?name=metabase-${{ matrix.edition }}-uberjar&per_page=100&page=$current_page")
-
-
-            zip=$(echo $artifacts | jq '[.artifacts[] | {url: .url, dl: .archive_download_url, run: .workflow_run}]' \
-            | jq -c --arg COMMIT "$parent_commit" '[.[] | select(.run.head_sha | contains($COMMIT))'.dl] | jq '.[0] | select (.!=null)')
-
-            if [[ $zip ]]; then
-                echo "Found metabase-${{ matrix.edition }} uberjar for '$parent_commit'!"
-
-                if [[ "${{ matrix.edition }}" == "ee" ]]; then
-                  echo "ee_dl=$(echo $zip)" >> $GITHUB_OUTPUT
-                else
-                  echo "oss_dl=$(echo $zip)" >> $GITHUB_OUTPUT
-                fi
-            elif [[ $current_page -le 3 ]]; then
-                echo "Didn't find the artifact for '$parent_commit' on the page: $current_page."
-                current_page=$((++current_page))
-                get_download_link $current_commit $current_page
-            else
-              current_commit=$parent_commit
-              echo "Switching to a new parent"
-              get_download_link $current_commit
-            fi
-          }
-
-          get_download_link $current_commit
-        shell: bash
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
   build:
-    needs: [download_uberjar, files-changed, e2e-matrix-builder]
+    needs: [files-changed, e2e-matrix-builder]
     if: |
       !cancelled() &&
       github.event.pull_request.draft == false &&
       needs.e2e-matrix-builder.result == 'success' &&
-      needs.download_uberjar.result == 'skipped' &&
       needs.files-changed.outputs.e2e_all == 'true'
     runs-on: ubuntu-22.04
     timeout-minutes: 25
@@ -155,17 +86,17 @@ jobs:
       - name: Upload JARs as artifact
         uses: actions/upload-artifact@v3
         with:
-          name: metabase-${{ matrix.edition }}-${{ github.sha }}-uberjar
+          name: metabase-${{ matrix.edition }}-${{ github.event.pull_request.head.sha || github.sha }}-uberjar
           path: |
             ./target/uberjar/metabase.jar
             ./COMMIT-ID
             ./SHA256.sum
 
   e2e-tests:
-    needs: [build, files-changed, test-run-id, download_uberjar, e2e-matrix-builder]
+    needs: [build, files-changed, test-run-id, e2e-matrix-builder]
     if: |
       !cancelled() &&
-      (needs.download_uberjar.result == 'success' || needs.build.result == 'success')
+      (needs.files-changed.outputs.e2e_specs == 'true' || needs.build.result == 'success')
     runs-on: ${{ matrix.runner }}
     timeout-minutes: 90
     name: e2e-tests-${{ matrix.name }}-${{ matrix.edition }}
@@ -190,6 +121,9 @@ jobs:
 
     steps:
       - uses: actions/checkout@v3
+        with:
+          # Important in case we need to download uberjar from previous commits!
+          fetch-depth: ${{ needs.build.result == 'success' && '' || 20 }}
 
       - name: Prepare Docker containers
         uses: ./.github/actions/e2e-prepare-containers
@@ -202,38 +136,11 @@ jobs:
           mysql: ${{ matrix.name != 'mongo'}}
           mongo: ${{ matrix.name == 'mongo'}}
 
-      - name: Download Metabase uberjar from a previously stored artifact
-        if: needs.download_uberjar.result == 'success'
-        run: |
-          if [[ "${{ matrix.edition }}" == "ee" ]]; then
-            DOWNLOAD_LINK="$(echo ${{ needs.download_uberjar.outputs.enterprise_download_link }})"
-          else
-            DOWNLOAD_LINK="$(echo ${{ needs.download_uberjar.outputs.oss_download_link }})"
-          fi
-
-          curl -sL \
-            -H "Accept: application/vnd.github+json" \
-            -H "Authorization: Bearer $GITHUB_TOKEN" \
-            -H "X-GitHub-Api-Version: 2022-11-28" \
-            $DOWNLOAD_LINK \
-            -o mb.zip
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        shell: bash
-      - name: Unzip Metabase artifact containing an uberjar
-        if: needs.download_uberjar.result == 'success'
-        run: unzip mb.zip
-
-      - uses: actions/download-artifact@v3
-        if: needs.build.result == 'success'
-        name: Retrieve uberjar artifact for ${{ matrix.edition }}
+      - name: Download Metabase ${{ matrix.edition }} uberjar
+        uses: ./.github/actions/e2e-download-uberjar
         with:
-          name: metabase-${{ matrix.edition }}-${{ github.sha }}-uberjar
-
-      - name: Get the version info
-        run: |
-          jar xf target/uberjar/metabase.jar version.properties
-          mv version.properties resources/
+          edition: ${{ matrix.edition }}
+          was-built: ${{ needs.build.result == 'success' }}
 
       - name: Prepare front-end environment
         uses: ./.github/actions/prepare-frontend
@@ -242,11 +149,13 @@ jobs:
         with:
           java-version: ${{ matrix.java-version }}
           distribution: "temurin"
+
       - name: Prepare Cypress environment
         id: cypress-prep
         uses: ./.github/actions/prepare-cypress
         with:
           is-replay-browser: ${{ github.event_name == 'schedule' }}
+
       - name: Run Snowplow micro
         uses: ./.github/actions/run-snowplow-micro
 
@@ -329,12 +238,10 @@ jobs:
           if-no-files-found: ignore
 
   e2e-tests-skipped-stub:
-    needs: [build, files-changed, download_uberjar, e2e-matrix-builder]
+    needs: [e2e-tests]
     if: |
       !cancelled() &&
-      needs.files-changed.outputs.e2e_all == 'false' &&
-      needs.build.result == 'skipped' &&
-      needs.download_uberjar.result == 'skipped'
+      needs.e2e-tests.result == 'skipped'
     runs-on: ${{ matrix.runner }}
     timeout-minutes: 5
     name: e2e-tests-${{ matrix.name }}-${{ matrix.edition }}
@@ -348,16 +255,17 @@ jobs:
   visual-regression-tests:
     runs-on: ubuntu-22.04
     timeout-minutes: 60
-    needs: [build, download_uberjar]
+    needs: [build]
     if: |
       !cancelled() &&
-      (needs.download_uberjar.result == 'success' || needs.build.result == 'success') &&
+      (needs.files-changed.outputs.e2e_specs == 'true' || needs.build.result == 'success') &&
       github.event_name != 'schedule'
     name: percy-visual-regression-tests
-    env:
-      MB_EDITION: ${{ matrix.edition }}
     steps:
       - uses: actions/checkout@v3
+        with:
+          # Important in case we need to download uberjar from previous commits!
+          fetch-depth: ${{ needs.build.result == 'success' && '' || 20 }}
 
       - name: Prepare Docker containers
         uses: ./.github/actions/e2e-prepare-containers
@@ -366,36 +274,11 @@ jobs:
           password: ${{ secrets.DOCKERHUB_TOKEN }}
           maildev: true
 
-      - name: Download Metabase uberjar from a previously stored artifact
-        if: needs.download_uberjar.result == 'success'
-        run: |
-          DOWNLOAD_LINK="$(echo ${{ needs.download_uberjar.outputs.enterprise_download_link }})"
-
-          curl -sL \
-            -H "Accept: application/vnd.github+json" \
-            -H "Authorization: Bearer $GITHUB_TOKEN" \
-            -H "X-GitHub-Api-Version: 2022-11-28" \
-            $DOWNLOAD_LINK \
-            -o mb.zip
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        shell: bash
-      - name: Unzip Metabase artifact containing an uberjar
-        if: needs.download_uberjar.result == 'success'
-        run: unzip mb.zip
-
-      - uses: actions/download-artifact@v3
-        if: needs.build.result == 'success'
-        name: Retrieve uberjar artifact for ee
+      - name: Download Metabase enterprise uberjar
+        uses: ./.github/actions/e2e-download-uberjar
         with:
-          name: metabase-ee-${{ github.sha }}-uberjar
-
-      - name: Get the version info
-        run: |
-          jar xf target/uberjar/metabase.jar version.properties
-          mv version.properties resources/
-
-
+          edition: 'ee'
+          was-built: ${{ needs.build.result == 'success' }}
 
       - name: Prepare front-end environment
         uses: ./.github/actions/prepare-frontend