Separate Replay E2E tests from the main workflow (#34895)

* Separate Replay E2E tests from the main workflow

* Fix workflow conditionals

.github/actions/prepare-cypress/action.yml

@@ -14,21 +14,21 @@ runs:
   steps:
     # Browsers
     - name: Install Chrome v111
-      if: inputs.is-replay-browser != true
+      if: ${{ inputs.is-replay-browser != 'true' }}
       uses: browser-actions/setup-chrome@v1
       with:
         # https://chromium.cypress.io/linux/stable/111.0.5563.146
         chrome-version: 1097615
       id: setup-chrome
     - name: Chrome install feedback
-      if: inputs.is-replay-browser != true
+      if: ${{ inputs.is-replay-browser != 'true' }}
       run: |
         echo Installed chromium version: ${{ steps.setup-chrome.outputs.chrome-version }}
         ${{ steps.setup-chrome.outputs.chrome-path }} --version
       shell: bash
 
     - name: Install Replay.io browser
-      if: inputs.is-replay-browser == true
+      if: ${{ inputs.is-replay-browser == 'true' }}
       run: npx @replayio/cypress install
       shell: bash
 

.github/workflows/e2e-tests-replay.yml

+name: E2E Tests using Replay.io browser
+
+on:
+  # We'll record runs using Replay.io and their browser on a schedule as an experiment
+  schedule:
+    - cron: '0 22 * * *'
+
+jobs:
+  test-run-id:
+    runs-on: ubuntu-22.04
+    outputs:
+      testRunId: ${{ steps.testRunId.outputs.testRunId }}
+    steps:
+      - id: testRunId
+        run: echo testRunId=$(npx uuid) >> "$GITHUB_OUTPUT"
+
+  e2e-matrix-builder:
+    runs-on: ubuntu-22.04
+    timeout-minutes: 5
+    outputs:
+      matrix: ${{ steps.e2e-matrix.outputs.matrix }}
+    steps:
+      - uses: actions/checkout@v3
+      - name: Generate matrix for E2E tests
+        id: e2e-matrix
+        uses: ./.github/actions/build-e2e-matrix
+
+
+  e2e-tests:
+    needs: [test-run-id, e2e-matrix-builder]
+    runs-on: ${{ matrix.runner }}
+    timeout-minutes: 90
+    name: e2e-tests-${{ matrix.name }}-${{ matrix.edition }}
+    env:
+      MB_EDITION: ${{ matrix.edition }}
+      DISPLAY: ""
+      QA_DB_ENABLED: true
+      # Any env starting with `CYPRESS_` will be available to all Cypress tests via `Cypress.env()`
+      # Example: you can get `CYPRESS_FOO` with `Cypress.env("FOO")`
+      CYPRESS_ALL_FEATURES_TOKEN: ${{ secrets.ENTERPRISE_TOKEN }}
+      CYPRESS_NO_FEATURES_TOKEN: ${{ secrets.E2E_STARTER_TOKEN }}
+      MB_SNOWPLOW_AVAILABLE: true
+      MB_SNOWPLOW_URL: "http://localhost:9090" # Snowplow micro
+      TZ: US/Pacific # to make node match the instance tz
+      TERM: xterm
+      CYPRESS_REPLAYIO_ENABLED: 1
+      RECORD_REPLAY_METADATA_FILE: /tmp/replay-metadata.json
+      RECORD_REPLAY_METADATA_TEST_RUN_ID: ${{ needs.test-run-id.outputs.testRunId }}
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJSON(needs.e2e-matrix-builder.outputs.matrix) }}
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          # Important in case we need to download uberjar from previous commits!
+          fetch-depth: 20
+
+      - name: Prepare Docker containers
+        uses: ./.github/actions/e2e-prepare-containers
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+          maildev: true
+          openldap: ${{ startsWith(matrix.name, 'admin') }}
+          postgres: ${{ matrix.name != 'mongo'}}
+          mysql: ${{ matrix.name != 'mongo'}}
+          mongo: ${{ matrix.name == 'mongo'}}
+
+      - name: Download Metabase ${{ matrix.edition }} uberjar
+        uses: ./.github/actions/e2e-download-uberjar
+        with:
+          edition: ${{ matrix.edition }}
+
+      - name: Prepare front-end environment
+        uses: ./.github/actions/prepare-frontend
+      - name: Prepare JDK ${{ matrix.java-version }}
+        uses: actions/setup-java@v3
+        with:
+          java-version: ${{ matrix.java-version }}
+          distribution: "temurin"
+
+      - name: Prepare Cypress environment
+        id: cypress-prep
+        uses: ./.github/actions/prepare-cypress
+        with:
+          is-replay-browser: true
+
+      - name: Run Snowplow micro
+        uses: ./.github/actions/run-snowplow-micro
+
+      - name: Run OSS-specific Cypress tests
+        if: matrix.edition == 'oss' && github.event_name != 'schedule'
+        run: |
+          yarn run test-cypress-run \
+          --env grepTags=@OSS,grepOmitFiltered=true \
+          --spec './e2e/test/scenarios/**/*.cy.spec.js' \
+          --browser "replay-chromium"
+
+      - name: Run slow and resource-intensive Cypress tests
+        if: matrix.name == 'slow' && github.event_name != 'schedule'
+        run: |
+          yarn run test-cypress-run \
+          --env grepTags="@slow",grepOmitFiltered=true \
+          --spec './e2e/test/scenarios/**/*.cy.spec.js' \
+          --browser "replay-chromium"
+
+      - name: Run EE Cypress tests on ${{ matrix.name }}
+        if: matrix.context == 'folder' && github.event_name != 'schedule'
+        run: |
+          yarn run test-cypress-run \
+          --env grepTags="-@slow+-@mongo --@quarantine",grepOmitFiltered=true \
+          --folder ${{ matrix.name }} \
+          --browser "replay-chromium"
+
+      - name: Run E2E tests that depend on Mongo
+        if: matrix.name == 'mongo'
+        run: |
+          yarn run test-cypress-run \
+          --env grepTags="@mongo --@quarantine",grepOmitFiltered=true \
+          --spec './e2e/test/scenarios/**/*.cy.spec.js' \
+          --browser "replay-chromium"
+        env:
+          CYPRESS_QA_DB_MONGO: true
+
+      - name: Upload Replay.io recordings
+        if: github.event_name == 'schedule' && !cancelled()
+        uses: replayio/action-upload@v0.5.1
+        with:
+          api-key: ${{ secrets.REPLAY_IO_TOKEN }}
+          public: false
+
+      - name: Upload Cypress Artifacts upon failure
+        uses: actions/upload-artifact@v3
+        if: failure()
+        with:
+          name: cypress-recording-${{ matrix.name }}-${{ matrix.edition }}
+          path: |
+            ./cypress
+            ./logs/test.log
+          if-no-files-found: ignore

.github/workflows/e2e-tests.yml

 name: E2E Tests
 
 on:
-  # We'll record runs using Replay.io and their browser on a schedule as an experiment
-  schedule:
-    - cron: '0 22 * * *'
   push:
     branches:
       - "master"
@@ -16,13 +13,6 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  test-run-id:
-    runs-on: ubuntu-22.04
-    outputs:
-      testRunId: ${{ steps.testRunId.outputs.testRunId }}
-    steps:
-      - id: testRunId
-        run: echo testRunId=$(npx uuid) >> "$GITHUB_OUTPUT"
   files-changed:
     name: Check which files changed
     runs-on: ubuntu-22.04
@@ -93,7 +83,7 @@ jobs:
             ./SHA256.sum
 
   e2e-tests:
-    needs: [build, files-changed, test-run-id, e2e-matrix-builder]
+    needs: [build, files-changed, e2e-matrix-builder]
     if: |
       !cancelled() &&
       (needs.files-changed.outputs.e2e_specs == 'true' || needs.build.result == 'success')
@@ -153,8 +143,6 @@ jobs:
       - name: Prepare Cypress environment
         id: cypress-prep
         uses: ./.github/actions/prepare-cypress
-        with:
-          is-replay-browser: ${{ github.event_name == 'schedule' }}
 
       - name: Run Snowplow micro
         uses: ./.github/actions/run-snowplow-micro
@@ -193,40 +181,6 @@ jobs:
         env:
           CYPRESS_QA_DB_MONGO: true
 
-      # REPLAY.IO specific jobs
-      - name: Install Replay.io browser
-        if: ${{ github.event_name == 'schedule' }}
-        run: npx @replayio/cypress install
-
-      - name: Run OSS-specific Cypress tests using Replay.io browser
-        if: matrix.edition == 'oss' && github.event_name == 'schedule'
-        run: |
-          yarn run test-cypress-run \
-          --env grepTags=@OSS,grepOmitFiltered=true \
-          --spec './e2e/test/scenarios/**/*.cy.spec.js' \
-          --browser "replay-chromium"
-          CYPRESS_REPLAYIO_ENABLED: 1
-          RECORD_REPLAY_METADATA_FILE: /tmp/replay-metadata.json
-          RECORD_REPLAY_METADATA_TEST_RUN_ID: ${{ needs.test-run-id.outputs.testRunId }}
-
-      - name: Run EE Cypress tests on ${{ matrix.name }} using Replay.io browser
-        if: matrix.context == 'folder' && github.event_name == 'schedule'
-        run: |
-          yarn run test-cypress-run \
-          --env grepTags="-@slow+-@mongo --@quarantine,grepOmitFiltered=true" \
-          --folder ${{ matrix.name }} \
-          --browser "replay-chromium"
-          CYPRESS_REPLAYIO_ENABLED: 1
-          RECORD_REPLAY_METADATA_FILE: /tmp/replay-metadata.json
-          RECORD_REPLAY_METADATA_TEST_RUN_ID: ${{ needs.test-run-id.outputs.testRunId }}
-
-      - name: Upload Replay.io recordings
-        if: github.event_name == 'schedule' && !cancelled()
-        uses: replayio/action-upload@v0.5.1
-        with:
-          api-key: ${{ secrets.REPLAY_IO_TOKEN }}
-          public: false
-
       - name: Upload Cypress Artifacts upon failure
         uses: actions/upload-artifact@v3
         if: failure()