[E2E] Cypress Replay.io Integration (#29787)

* Install `replay.io` library

* Register `replay.io` in the Cypress config

* Run E2E tests using Replay chromium browser but only in CI

* Upload Replay.io recordings to the dashboard

* Manually install Replay.io browser

* Always upload recordings

* Pass in a custom test run id

* Disable asserts and send replays to a separate team

* Upload OSS recordings as well

* Use specific Ubuntu version

* Record and run Replay.io on `master` only

* Do not toggle CI browsers in the config

* Test run: pass `replay-chromium` browser as CLI flag in a run

* Fix multi-line command

* Use replay plugin conditionally

* Set the flag correctly

* Require node process

* Remove sourcemap vars

* Record using replay.io on schedule

* Explicitly name replay runs

---------

Co-authored-by: Jaril <jarilvalenciano@gmail.com>

.github/workflows/e2e-tests.yml

 name: E2E Tests
 
 on:
+  # We'll record runs using Replay.io and their browser on a schedule as an experiment
+  schedule:
+    - cron: '0 */8 * * *'
   push:
     branches:
       - "master"
@@ -13,6 +16,13 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  test-run-id:
+    runs-on: ubuntu-22.04
+    outputs:
+      testRunId: ${{ steps.testRunId.outputs.testRunId }}
+    steps:
+      - id: testRunId
+        run: echo testRunId=$(npx uuid) >> "$GITHUB_OUTPUT"
   files-changed:
     name: Check which files changed
     runs-on: ubuntu-22.04
@@ -55,7 +65,7 @@ jobs:
         uses: ./.github/actions/prepare-uberjar-artifact
 
   e2e-tests:
-    needs: [build, files-changed]
+    needs: [build, files-changed, test-run-id]
     if: |
       always() &&
       needs.files-changed.outputs.e2e_all == 'true' &&
@@ -166,7 +176,7 @@ jobs:
           mv version.properties resources/
 
       - name: Run OSS-specific Cypress tests
-        if: matrix.edition == 'oss'
+        if: matrix.edition == 'oss' && github.event_name != 'schedule'
         run: |
           yarn run test-cypress-run \
           --env grepTags=@OSS \
@@ -175,7 +185,7 @@ jobs:
           TERM: xterm
 
       - name: Run EE Cypress tests on ${{ matrix.folder }}
-        if: matrix.edition == 'ee'
+        if: matrix.edition == 'ee' && github.event_name != 'schedule'
         run: |
           yarn run test-cypress-run \
           --env grepTags="-@quarantine" \
@@ -183,6 +193,46 @@ jobs:
         env:
           TERM: xterm
 
+      # REPLAY.IO specific jobs
+      - name: Install Replay.io browser
+        if: ${{ github.event_name == 'schedule' }}
+        run: npx @replayio/cypress install
+
+      - name: Run OSS-specific Cypress tests using Replay.io browser
+        if: matrix.edition == 'oss' && github.event_name == 'schedule'
+        run: |
+          yarn run test-cypress-run \
+          --env grepTags=@OSS \
+          --spec './e2e/test/scenarios/**/*.cy.spec.js' \
+          --browser "replay-chromium"
+        env:
+          TERM: xterm
+          REPLAYIO_ENABLED: 1
+          RECORD_REPLAY_METADATA_FILE: /tmp/replay-metadata.json
+          RECORD_REPLAY_METADATA_TEST_RUN_ID: ${{ needs.test-run-id.outputs.testRunId }}
+          RECORD_REPLAY_DISABLE_ASSERTS: 1
+
+      - name: Run EE Cypress tests on ${{ matrix.folder }} using Replay.io browser
+        if: matrix.edition == 'ee' && github.event_name == 'schedule'
+        run: |
+          yarn run test-cypress-run \
+          --env grepTags="-@quarantine" \
+          --folder ${{ matrix.folder }} \
+          --browser "replay-chromium"
+        env:
+          TERM: xterm
+          REPLAYIO_ENABLED: 1
+          RECORD_REPLAY_METADATA_FILE: /tmp/replay-metadata.json
+          RECORD_REPLAY_METADATA_TEST_RUN_ID: ${{ needs.test-run-id.outputs.testRunId }}
+          RECORD_REPLAY_DISABLE_ASSERTS: 1
+
+      - name: Upload Replay.io recordings
+        if: ${{ github.event_name == 'schedule' }}
+        uses: replayio/action-upload@v0.4.7
+        with:
+          api-key: rwk_gXbvYctIcR6RZyEzUvby3gtkO4esrB2L321lkY8FSuQ
+          public: true
+
       - name: Upload Cypress Artifacts upon failure
         uses: actions/upload-artifact@v3
         if: failure()
@@ -239,6 +289,7 @@ jobs:
     runs-on: ubuntu-22.04
     timeout-minutes: 45
     needs: build
+    if: ${{ github.event_name != 'schedule' }}
     name: percy-visual-regression-tests
     env:
       MB_EDITION: ${{ matrix.edition }}

e2e/support/config.js

 import * as dbTasks from "./db_tasks";
+const replay = require("@replayio/cypress");
 const { verifyDownloadTasks } = require("cy-verify-downloads");
 const {
   NodeModulesPolyfillPlugin,
@@ -22,6 +23,8 @@ const isQaDatabase = process.env["QA_DB_ENABLED"];
 const sourceVersion = process.env["CROSS_VERSION_SOURCE"];
 const targetVersion = process.env["CROSS_VERSION_TARGET"];
 
+const runWithReplay = process.env["REPLAYIO_ENABLED"];
+
 // This function is called when a project is opened or re-opened (e.g. due to
 // the project's config changing)
 
@@ -99,6 +102,10 @@ const defaultConfig = {
 
     require("@cypress/grep/src/plugin")(config);
 
+    if (runWithReplay) {
+      replay.default(on, config);
+    }
+
     return config;
   },
   supportFile: "e2e/support/cypress.js",

e2e/support/cypress.js

@@ -11,6 +11,12 @@ import "@cypress/skip-test/support";
 import "@percy/cypress";
 import "./commands";
 
+const { env } = require("node:process");
+const runWithReplay = env["REPLAYIO_ENABLED"];
+
+if (runWithReplay) {
+  require("@replayio/cypress/support");
+}
 require("cy-verify-downloads").addCustomCommand();
 
 Cypress.on("uncaught:exception", (err, runnable) => false);

package.json

@@ -128,6 +128,7 @@
     "@percy/cli": "^1.23.0",
     "@percy/cypress": "^3.1.2",
     "@pmmmwh/react-refresh-webpack-plugin": "0.5.10",
+    "@replayio/cypress": "^0.5.15",
     "@sinonjs/fake-timers": "^9.1.2",
     "@storybook/addon-a11y": "^6.5.16",
     "@storybook/addon-actions": "6.5.15",