[E2E] PoC for cross-version testing (#24617)

* Dummy setup

* Wait for the first job to finish

* Use volumes

* Use docker

* Run Cypress

* Initial setup

* Full upgrade test

* Try using maps in a matrix

* Fix breaking string

* Let all matrix workflows finish

* Update `matrix` naming

- `upgrade` is too specific.
- `version` gives us the flexibility to go in both directions (upgrade/downgrade)

* Rename the job to `cross-version-testing`

* DRY setup code

* Enable cross-version testing for EE edition

* Remove test retries

* Pass the secret to the Docker container

* Spice things up

* Schedule a job every Sunday at 9

.github/workflows/e2e-cross-version.yml

+name: E2E Cross-version Tests
+
+on:
+  schedule:
+    - cron: '0 9 * * 0'
+
+jobs:
+  cross-version-testing:
+    runs-on: ubuntu-20.04
+    timeout-minutes: 60
+    strategy:
+      matrix:
+        version: [
+          # OSS upgrade
+          { source: v0.41.3.1, target: v0.42.2 },
+          # EE upgrade
+          { source: v1.43.4, target: v1.44.0 },
+          # EE downgrade
+          { source: v1.42.0, target: v1.41.3.1 },
+          # Cross-edition upgrade
+          { source: v0.41.3.1, target: v1.42.2 }
+        ]
+      fail-fast: false
+    env:
+      CROSS_VERSION_SOURCE: ${{ matrix.version.source }}
+      CROSS_VERSION_TARGET: ${{ matrix.version.target }}
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+    - name: Set Metabase Enterprise License
+      run: |
+        echo "MB_PREMIUM_EMBEDDING_TOKEN=${{ secrets.ENTERPRISE_TOKEN }}" >> $GITHUB_ENV
+      if: startsWith(matrix.version.source, 'v1') || startsWith(matrix.version.target, 'v1')
+    - name: Set source Docker image to OSS
+      run: |
+        echo "DOCKER_SOURCE_IMAGE=metabase/metabase:${{ matrix.version.source }}" >> $GITHUB_ENV
+      if: startsWith(matrix.version.source, 'v0')
+    - name: Set source Docker image to EE
+      run: |
+        echo "DOCKER_SOURCE_IMAGE=metabase/metabase-enterprise:${{ matrix.version.source }}" >> $GITHUB_ENV
+      if: startsWith(matrix.version.source, 'v1')
+    - name: Prepare front-end environment
+      uses: ./.github/actions/prepare-frontend
+    - name: Start Metabase ${{ matrix.version.source }}
+      run: |
+        docker run -d \
+        -v $PWD/my-metabase:/metabase.db \
+        -p 3000:3000 \
+        -e MB_PREMIUM_EMBEDDING_TOKEN \
+        --name metabase-${{ matrix.version.source }} \
+        ${{ env.DOCKER_SOURCE_IMAGE }}
+    - name: Wait for Metabase to start on port 3000
+      run: while ! curl -s 'http://localhost:3000/api/health' | grep '{"status":"ok"}'; do sleep 1; done
+    - name: Prepare Cypress environment
+      uses: ./.github/actions/prepare-cypress
+    - name: Run Cypress on the source
+      run: |
+        yarn cypress run \
+          --browser chrome \
+          --config-file frontend/test/metabase/scenarios/cross-version/source/cypress.json \
+          --spec frontend/test/metabase/scenarios/cross-version/source/**/*.cy.spec.js
+    - name: Stop Metabase ${{ matrix.version.source }}
+      run: docker stop metabase-${{ matrix.version.source }}
+
+    - name: Set target Docker image to OSS
+      run: |
+        echo "DOCKER_TARGET_IMAGE=metabase/metabase:${{ matrix.version.source }}" >> $GITHUB_ENV
+      if: startsWith(matrix.version.source, 'v0')
+    - name: Set target Docker image to EE
+      run: |
+        echo "DOCKER_TARGET_IMAGE=metabase/metabase-enterprise:${{ matrix.version.source }}" >> $GITHUB_ENV
+      if: startsWith(matrix.version.source, 'v1')
+    - name: Start Metabase ${{ matrix.version.target }}
+      run: |
+        docker run -d \
+          -v $PWD/my-metabase:/metabase.db \
+          -p 3001:3000 \
+          -e MB_PREMIUM_EMBEDDING_TOKEN \
+          --name metabase-${{ matrix.version.target }} \
+          ${{ env.DOCKER_TARGET_IMAGE }}
+    - name: Wait for Metabase to start on port 3001
+      run: while ! curl -s 'http://localhost:3001/api/health' | grep '{"status":"ok"}'; do sleep 1; done
+    - name: Run Cypress on the target
+      run: |
+        yarn cypress run \
+          --browser chrome \
+          --config-file frontend/test/metabase/scenarios/cross-version/target/cypress.json \
+          --spec frontend/test/metabase/scenarios/cross-version/target/**/*.cy.spec.js
+    - name: Upload Cypress Artifacts upon failure
+      uses: actions/upload-artifact@v2
+      if: failure()
+      with:
+        name: cypress-artifacts
+        path: |
+          ./cypress
+        if-no-files-found: ignore

frontend/test/__support__/e2e/cypress-plugins.js

@@ -23,6 +23,9 @@ const snowplowMicroUrl = process.env["MB_SNOWPLOW_URL"];
 
 const isQaDatabase = process.env["QA_DB_ENABLED"];
 
+const sourceVersion = process.env["CROSS_VERSION_SOURCE"];
+const targetVersion = process.env["CROSS_VERSION_TARGET"];
+
 // This function is called when a project is opened or re-opened (e.g. due to
 // the project's config changing)
 
@@ -72,6 +75,8 @@ module.exports = (on, config) => {
   config.env.HAS_ENTERPRISE_TOKEN = hasEnterpriseToken;
   config.env.HAS_SNOWPLOW_MICRO = hasSnowplowMicro;
   config.env.SNOWPLOW_MICRO_URL = snowplowMicroUrl;
+  config.env.SOURCE_VERSION = sourceVersion;
+  config.env.TARGET_VERSION = targetVersion;
 
   return config;
 };

frontend/test/metabase/scenarios/cross-version/source/cross-version-source-helpers.js

+export const version = Cypress.env("SOURCE_VERSION");
+
+export function setupLanguage() {
+  // Make sure English is the default selected language
+  cy.findByText("English")
+    .should("have.css", "background-color")
+    .and("eq", "rgb(80, 158, 227)");
+
+  cy.button("Next").click();
+  cy.findByText("Your language is set to English");
+}
+
+export function setupInstance(version) {
+  const companyLabel =
+    version === "v0.41.3.1"
+      ? "Your company or team name"
+      : "Company or team name";
+
+  const finalSetupButton = version === "v0.41.3.1" ? "Next" : "Finish";
+
+  cy.findByLabelText("First name").type("Superuser");
+  cy.findByLabelText("Last name").type("Tableton");
+  cy.findByLabelText("Email").type("admin@metabase.test");
+  cy.findByLabelText(companyLabel).type("Metabase");
+  cy.findByLabelText("Create a password").type("12341234");
+  cy.findByLabelText("Confirm your password").type("12341234");
+  cy.button("Next").click();
+  cy.findByText("Hi, Superuser. Nice to meet you!");
+
+  cy.findByText("I'll add my data later").click();
+  cy.findByText("I'll add my own data later");
+
+  // Collection defaults to on and describes data collection
+  cy.findByText("All collection is completely anonymous.");
+  // turn collection off, which hides data collection description
+  cy.findByLabelText(
+    "Allow Metabase to anonymously collect usage events",
+  ).click();
+  cy.findByText("All collection is completely anonymous.").should("not.exist");
+  cy.findByText(finalSetupButton).click();
+  cy.findByText("Take me to Metabase").click();
+
+  cy.location("pathname").should("eq", "/");
+  cy.contains("Reviews");
+}

frontend/test/metabase/scenarios/cross-version/source/cypress.json

+{
+  "testFiles": "**/*.cy.spec.js",
+  "integrationFolder": "frontend/test/metabase/scenarios/cross-version/source/",
+  "pluginsFile": "frontend/test/__support__/e2e/cypress-plugins.js",
+  "supportFile": "frontend/test/__support__/e2e/cypress.js",
+  "baseUrl": "http://localhost:3000",
+  "videoUploadOnPasses": false,
+  "chromeWebSecurity": false,
+  "modifyObstructiveCode": false,
+  "viewportHeight": 800,
+  "viewportWidth": 1280
+}

frontend/test/metabase/scenarios/cross-version/source/setup.cy.spec.js

+import {
+  version,
+  setupLanguage,
+  setupInstance,
+} from "./cross-version-source-helpers";
+
+describe(`setup on ${version}`, () => {
+  it("should set up metabase", () => {
+    cy.visit("/");
+    // It redirects to the setup page
+    cy.location("pathname").should("eq", "/setup");
+    cy.findByText("Welcome to Metabase");
+    cy.findByText("Let's get started").click();
+
+    setupLanguage();
+    setupInstance(version);
+
+    // Quick and dirty sanity check for EE version
+    // TODO: Remove or refactor properly
+    if (version.startsWith("v1")) {
+      cy.visit("/admin/settings/license");
+      cy.findByPlaceholderText("Using MB_PREMIUM_EMBEDDING_TOKEN").should(
+        "be.disabled",
+      );
+    } else {
+      cy.visit("/admin");
+      cy.icon("store");
+    }
+  });
+});

frontend/test/metabase/scenarios/cross-version/target/cross-version-target-helpers.js

+export const version = Cypress.env("TARGET_VERSION");

frontend/test/metabase/scenarios/cross-version/target/cypress.json

+{
+  "testFiles": "**/*.cy.spec.js",
+  "integrationFolder": "frontend/test/metabase/scenarios/cross-version/target/",
+  "pluginsFile": "frontend/test/__support__/e2e/cypress-plugins.js",
+  "supportFile": "frontend/test/__support__/e2e/cypress.js",
+  "baseUrl": "http://localhost:3001",
+  "videoUploadOnPasses": false,
+  "chromeWebSecurity": false,
+  "modifyObstructiveCode": false,
+  "viewportHeight": 800,
+  "viewportWidth": 1280
+}

frontend/test/metabase/scenarios/cross-version/target/smoke.cy.spec.js

+import { version } from "./cross-version-target-helpers";
+
+describe(`smoke test the migration to the version ${version}`, () => {
+  it("should already be set up", () => {
+    cy.visit("/");
+    cy.findByText("Sign in to Metabase");
+  });
+});