diff --git a/.github/actions/e2e-download-uberjar/action.yml b/.github/actions/e2e-download-uberjar/action.yml new file mode 100644 index 0000000000000000000000000000000000000000..c2ec019067ffbb311bf14bd995b30761693ca268 --- /dev/null +++ b/.github/actions/e2e-download-uberjar/action.yml @@ -0,0 +1,88 @@ +name: Download uberjar for E2E tests +description: Download either a freshly built artifact or the one built in one of the previous commits. +inputs: + edition: + description: Metabase edition. + required: true + was-built: + description: Was the artifact built during the current workflof run? + required: true + +runs: + using: "composite" + steps: + - uses: actions/download-artifact@v3 + if: ${{ inputs.was-built == 'true' }} + name: Retrieve uberjar artifact for ${{ inputs.edition }} + with: + name: metabase-${{ inputs.edition }}-${{ github.event.pull_request.head.sha || github.sha }}-uberjar + + - name: Download previously stored uberjar + if: ${{ inputs.was-built == 'false' }} + uses: actions/github-script@v6 + with: + script: | # js + const fs = require('fs'); + const execSync = require("child_process").execSync; + + const getCommit = (ref = "HEAD") => { + const [commit] = execSync(`git rev-parse ${ref}`, { encoding: "utf8" }).split( + "\n", + ); + return commit; + }; + + const baseConfig = { + owner: context.repo.owner, + repo: context.repo.repo, + }; + + async function getArtifact(commit, depth = 0) { + if (depth > 20) { + throw new Error("Couldn't find the artifact!"); + } + + const artifactName = `metabase-${{ inputs.edition }}-${commit}-uberjar`; + + const { + data: { artifacts }, + } = await github.rest.actions.listArtifactsForRepo({ + ...baseConfig, + name: artifactName, + per_page: 1, + }); + + const [artifact] = artifacts; + + if (!artifact) { + const parentCommit = getCommit(`${commit}^`); + console.log(`Didn't find ${artifactName}.\n`); + console.log(`Switching to the parent commit: ${parentCommit}`); + return getArtifact(parentCommit, depth + 1); + } else { + return artifact; + } + } + + const currentCommit = "${{ github.event.pull_request.head.sha || github.sha }}"; + const { id } = await getArtifact(currentCommit); + + const download = await github.rest.actions.downloadArtifact({ + ...baseConfig, + artifact_id: id, + archive_format: 'zip', + }); + + fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/mb.zip`, Buffer.from(download.data)); + + - name: Unzip Metabase artifact containing an uberjar + if: ${{ inputs.was-built == 'false' }} + run: unzip mb.zip + shell: bash + + - name: Get the version info + run: | + jar xf target/uberjar/metabase.jar version.properties + cat version.properties + mv version.properties resources/ + shell: bash diff --git a/.github/workflows/e2e-stress-test-flake-fix.yml b/.github/workflows/e2e-stress-test-flake-fix.yml index 5a6d6991fceb50009296f1574607fc598c864f7a..5807bbd2bb380a399eb087244a0aa5b47479a121 100644 --- a/.github/workflows/e2e-stress-test-flake-fix.yml +++ b/.github/workflows/e2e-stress-test-flake-fix.yml @@ -3,10 +3,6 @@ name: E2E Stress Test Flake Fix on: workflow_dispatch: inputs: - artifact: - description: 'Artifact ID' - type: string - required: true spec: description: 'Relative path of the target spec' type: string @@ -32,6 +28,9 @@ jobs: TZ: US/Pacific # to make node match the instance tz steps: - uses: actions/checkout@v3 + with: + # Important because we need previous commits hashes to find and download the uberjar! + fetch-depth: 20 - name: Prepare Docker containers uses: ./.github/actions/e2e-prepare-containers @@ -44,19 +43,11 @@ jobs: mysql: true mongo: true - - name: Download Metabase uberjar from a previously stored artifact - run: | - curl -L \ - -H "Accept: application/vnd.github+json" \ - -H "Authorization: Bearer $GITHUB_TOKEN" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - https://api.github.com/repos/metabase/metabase/actions/artifacts/$ARTIFACT_ID/zip \ - -o mb.zip - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - ARTIFACT_ID: ${{ github.event.inputs.artifact }} - - name: Unzip Metabase artifact containing an uberjar - run: unzip mb.zip + - name: Download Metabase ${{ matrix.edition }} uberjar + uses: ./.github/actions/e2e-download-uberjar + with: + edition: 'ee' + - name: Prepare front-end environment uses: ./.github/actions/prepare-frontend - name: Prepare JDK 11 @@ -69,10 +60,6 @@ jobs: uses: ./.github/actions/prepare-cypress - name: Run Snowplow micro uses: ./.github/actions/run-snowplow-micro - - name: Get Metabase version info - run: | - jar xf target/uberjar/metabase.jar version.properties - mv version.properties resources/ - name: Stress-test ${{ github.event.inputs.spec }} ${{ github.event.inputs.burn_in }} times run: | diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index 6f4a271837f45e5d233598c3c2a9b3e58265d579..1b4fe38b2259e44c92a8a1f094bed4d2712ff958 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -50,81 +50,12 @@ jobs: id: e2e-matrix uses: ./.github/actions/build-e2e-matrix - download_uberjar: - runs-on: ubuntu-22.04 - timeout-minutes: 10 - needs: [files-changed, e2e-matrix-builder] - if: | - !cancelled() && - github.event.pull_request.draft == false && - needs.e2e-matrix-builder.result == 'success' && - needs.files-changed.outputs.e2e_specs == 'true' && - needs.files-changed.outputs.e2e_all != 'true' - strategy: - matrix: - edition: [oss, ee] - outputs: - enterprise_download_link: ${{ steps.zip_url.outputs.ee_dl }} - oss_download_link: ${{ steps.zip_url.outputs.oss_dl }} - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: Artifact zip URL - id: zip_url - run: | - current_commit="HEAD" - - get_parent () { - parent_commit=$(git rev-parse $1^) - echo $parent_commit - } - - get_download_link () { - parent_commit=$(get_parent $1) - current_page=${2:-1} - - artifacts=$(curl -sL \ - -H "Accept: application/vnd.github+json" \ - -H "Authorization: Bearer $GITHUB_TOKEN" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - "https://api.github.com/repos/metabase/metabase/actions/artifacts?name=metabase-${{ matrix.edition }}-uberjar&per_page=100&page=$current_page") - - - zip=$(echo $artifacts | jq '[.artifacts[] | {url: .url, dl: .archive_download_url, run: .workflow_run}]' \ - | jq -c --arg COMMIT "$parent_commit" '[.[] | select(.run.head_sha | contains($COMMIT))'.dl] | jq '.[0] | select (.!=null)') - - if [[ $zip ]]; then - echo "Found metabase-${{ matrix.edition }} uberjar for '$parent_commit'!" - - if [[ "${{ matrix.edition }}" == "ee" ]]; then - echo "ee_dl=$(echo $zip)" >> $GITHUB_OUTPUT - else - echo "oss_dl=$(echo $zip)" >> $GITHUB_OUTPUT - fi - elif [[ $current_page -le 3 ]]; then - echo "Didn't find the artifact for '$parent_commit' on the page: $current_page." - current_page=$((++current_page)) - get_download_link $current_commit $current_page - else - current_commit=$parent_commit - echo "Switching to a new parent" - get_download_link $current_commit - fi - } - - get_download_link $current_commit - shell: bash - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - build: - needs: [download_uberjar, files-changed, e2e-matrix-builder] + needs: [files-changed, e2e-matrix-builder] if: | !cancelled() && github.event.pull_request.draft == false && needs.e2e-matrix-builder.result == 'success' && - needs.download_uberjar.result == 'skipped' && needs.files-changed.outputs.e2e_all == 'true' runs-on: ubuntu-22.04 timeout-minutes: 25 @@ -155,17 +86,17 @@ jobs: - name: Upload JARs as artifact uses: actions/upload-artifact@v3 with: - name: metabase-${{ matrix.edition }}-${{ github.sha }}-uberjar + name: metabase-${{ matrix.edition }}-${{ github.event.pull_request.head.sha || github.sha }}-uberjar path: | ./target/uberjar/metabase.jar ./COMMIT-ID ./SHA256.sum e2e-tests: - needs: [build, files-changed, test-run-id, download_uberjar, e2e-matrix-builder] + needs: [build, files-changed, test-run-id, e2e-matrix-builder] if: | !cancelled() && - (needs.download_uberjar.result == 'success' || needs.build.result == 'success') + (needs.files-changed.outputs.e2e_specs == 'true' || needs.build.result == 'success') runs-on: ${{ matrix.runner }} timeout-minutes: 90 name: e2e-tests-${{ matrix.name }}-${{ matrix.edition }} @@ -190,6 +121,9 @@ jobs: steps: - uses: actions/checkout@v3 + with: + # Important in case we need to download uberjar from previous commits! + fetch-depth: ${{ needs.build.result == 'success' && '' || 20 }} - name: Prepare Docker containers uses: ./.github/actions/e2e-prepare-containers @@ -202,38 +136,11 @@ jobs: mysql: ${{ matrix.name != 'mongo'}} mongo: ${{ matrix.name == 'mongo'}} - - name: Download Metabase uberjar from a previously stored artifact - if: needs.download_uberjar.result == 'success' - run: | - if [[ "${{ matrix.edition }}" == "ee" ]]; then - DOWNLOAD_LINK="$(echo ${{ needs.download_uberjar.outputs.enterprise_download_link }})" - else - DOWNLOAD_LINK="$(echo ${{ needs.download_uberjar.outputs.oss_download_link }})" - fi - - curl -sL \ - -H "Accept: application/vnd.github+json" \ - -H "Authorization: Bearer $GITHUB_TOKEN" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - $DOWNLOAD_LINK \ - -o mb.zip - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - shell: bash - - name: Unzip Metabase artifact containing an uberjar - if: needs.download_uberjar.result == 'success' - run: unzip mb.zip - - - uses: actions/download-artifact@v3 - if: needs.build.result == 'success' - name: Retrieve uberjar artifact for ${{ matrix.edition }} + - name: Download Metabase ${{ matrix.edition }} uberjar + uses: ./.github/actions/e2e-download-uberjar with: - name: metabase-${{ matrix.edition }}-${{ github.sha }}-uberjar - - - name: Get the version info - run: | - jar xf target/uberjar/metabase.jar version.properties - mv version.properties resources/ + edition: ${{ matrix.edition }} + was-built: ${{ needs.build.result == 'success' }} - name: Prepare front-end environment uses: ./.github/actions/prepare-frontend @@ -242,11 +149,13 @@ jobs: with: java-version: ${{ matrix.java-version }} distribution: "temurin" + - name: Prepare Cypress environment id: cypress-prep uses: ./.github/actions/prepare-cypress with: is-replay-browser: ${{ github.event_name == 'schedule' }} + - name: Run Snowplow micro uses: ./.github/actions/run-snowplow-micro @@ -329,12 +238,10 @@ jobs: if-no-files-found: ignore e2e-tests-skipped-stub: - needs: [build, files-changed, download_uberjar, e2e-matrix-builder] + needs: [e2e-tests] if: | !cancelled() && - needs.files-changed.outputs.e2e_all == 'false' && - needs.build.result == 'skipped' && - needs.download_uberjar.result == 'skipped' + needs.e2e-tests.result == 'skipped' runs-on: ${{ matrix.runner }} timeout-minutes: 5 name: e2e-tests-${{ matrix.name }}-${{ matrix.edition }} @@ -348,16 +255,17 @@ jobs: visual-regression-tests: runs-on: ubuntu-22.04 timeout-minutes: 60 - needs: [build, download_uberjar] + needs: [build] if: | !cancelled() && - (needs.download_uberjar.result == 'success' || needs.build.result == 'success') && + (needs.files-changed.outputs.e2e_specs == 'true' || needs.build.result == 'success') && github.event_name != 'schedule' name: percy-visual-regression-tests - env: - MB_EDITION: ${{ matrix.edition }} steps: - uses: actions/checkout@v3 + with: + # Important in case we need to download uberjar from previous commits! + fetch-depth: ${{ needs.build.result == 'success' && '' || 20 }} - name: Prepare Docker containers uses: ./.github/actions/e2e-prepare-containers @@ -366,36 +274,11 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} maildev: true - - name: Download Metabase uberjar from a previously stored artifact - if: needs.download_uberjar.result == 'success' - run: | - DOWNLOAD_LINK="$(echo ${{ needs.download_uberjar.outputs.enterprise_download_link }})" - - curl -sL \ - -H "Accept: application/vnd.github+json" \ - -H "Authorization: Bearer $GITHUB_TOKEN" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - $DOWNLOAD_LINK \ - -o mb.zip - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - shell: bash - - name: Unzip Metabase artifact containing an uberjar - if: needs.download_uberjar.result == 'success' - run: unzip mb.zip - - - uses: actions/download-artifact@v3 - if: needs.build.result == 'success' - name: Retrieve uberjar artifact for ee + - name: Download Metabase enterprise uberjar + uses: ./.github/actions/e2e-download-uberjar with: - name: metabase-ee-${{ github.sha }}-uberjar - - - name: Get the version info - run: | - jar xf target/uberjar/metabase.jar version.properties - mv version.properties resources/ - - + edition: 'ee' + was-built: ${{ needs.build.result == 'success' }} - name: Prepare front-end environment uses: ./.github/actions/prepare-frontend