From 28fe410d9ab4774835e5f2a3c78560f301fd2f81 Mon Sep 17 00:00:00 2001
From: Nemanja Glumac <31325167+nemanjaglumac@users.noreply.github.com>
Date: Wed, 24 Aug 2022 20:29:11 +0200
Subject: [PATCH] [E2E] PoC for cross-version testing (#24617)
* Dummy setup
* Wait for the first job to finish
* Use volumes
* Use docker
* Run Cypress
* Initial setup
* Full upgrade test
* Try using maps in a matrix
* Fix breaking string
* Let all matrix workflows finish
* Update `matrix` naming
- `upgrade` is too specific.
- `version` gives us the flexibility to go in both directions (upgrade/downgrade)
* Rename the job to `cross-version-testing`
* DRY setup code
* Enable cross-version testing for EE edition
* Remove test retries
* Pass the secret to the Docker container
* Spice things up
* Schedule a job every Sunday at 9
---
.github/workflows/e2e-cross-version.yml | 97 +++++++++++++++++++
.../test/__support__/e2e/cypress-plugins.js | 5 +
.../source/cross-version-source-helpers.js | 45 +++++++++
.../cross-version/source/cypress.json | 12 +++
.../cross-version/source/setup.cy.spec.js | 30 ++++++
.../target/cross-version-target-helpers.js | 1 +
.../cross-version/target/cypress.json | 12 +++
.../cross-version/target/smoke.cy.spec.js | 8 ++
8 files changed, 210 insertions(+)
create mode 100644 .github/workflows/e2e-cross-version.yml
create mode 100644 frontend/test/metabase/scenarios/cross-version/source/cross-version-source-helpers.js
create mode 100644 frontend/test/metabase/scenarios/cross-version/source/cypress.json
create mode 100644 frontend/test/metabase/scenarios/cross-version/source/setup.cy.spec.js
create mode 100644 frontend/test/metabase/scenarios/cross-version/target/cross-version-target-helpers.js
create mode 100644 frontend/test/metabase/scenarios/cross-version/target/cypress.json
create mode 100644 frontend/test/metabase/scenarios/cross-version/target/smoke.cy.spec.js
diff --git a/.github/workflows/e2e-cross-version.yml b/.github/workflows/e2e-cross-version.yml
new file mode 100644
index 00000000000..55c07b34e5f
--- /dev/null
+++ b/.github/workflows/e2e-cross-version.yml
@@ -0,0 +1,97 @@
+name: E2E Cross-version Tests
+
+on:
+ schedule:
+ - cron: '0 9 * * 0'
+
+jobs:
+ cross-version-testing:
+ runs-on: ubuntu-20.04
+ timeout-minutes: 60
+ strategy:
+ matrix:
+ version: [
+ # OSS upgrade
+ { source: v0.41.3.1, target: v0.42.2 },
+ # EE upgrade
+ { source: v1.43.4, target: v1.44.0 },
+ # EE downgrade
+ { source: v1.42.0, target: v1.41.3.1 },
+ # Cross-edition upgrade
+ { source: v0.41.3.1, target: v1.42.2 }
+ ]
+ fail-fast: false
+ env:
+ CROSS_VERSION_SOURCE: ${{ matrix.version.source }}
+ CROSS_VERSION_TARGET: ${{ matrix.version.target }}
+
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v3
+ - name: Set Metabase Enterprise License
+ run: |
+ echo "MB_PREMIUM_EMBEDDING_TOKEN=${{ secrets.ENTERPRISE_TOKEN }}" >> $GITHUB_ENV
+ if: startsWith(matrix.version.source, 'v1') || startsWith(matrix.version.target, 'v1')
+ - name: Set source Docker image to OSS
+ run: |
+ echo "DOCKER_SOURCE_IMAGE=metabase/metabase:${{ matrix.version.source }}" >> $GITHUB_ENV
+ if: startsWith(matrix.version.source, 'v0')
+ - name: Set source Docker image to EE
+ run: |
+ echo "DOCKER_SOURCE_IMAGE=metabase/metabase-enterprise:${{ matrix.version.source }}" >> $GITHUB_ENV
+ if: startsWith(matrix.version.source, 'v1')
+ - name: Prepare front-end environment
+ uses: ./.github/actions/prepare-frontend
+ - name: Start Metabase ${{ matrix.version.source }}
+ run: |
+ docker run -d \
+ -v $PWD/my-metabase:/metabase.db \
+ -p 3000:3000 \
+ -e MB_PREMIUM_EMBEDDING_TOKEN \
+ --name metabase-${{ matrix.version.source }} \
+ ${{ env.DOCKER_SOURCE_IMAGE }}
+ - name: Wait for Metabase to start on port 3000
+ run: while ! curl -s 'http://localhost:3000/api/health' | grep '{"status":"ok"}'; do sleep 1; done
+ - name: Prepare Cypress environment
+ uses: ./.github/actions/prepare-cypress
+ - name: Run Cypress on the source
+ run: |
+ yarn cypress run \
+ --browser chrome \
+ --config-file frontend/test/metabase/scenarios/cross-version/source/cypress.json \
+ --spec frontend/test/metabase/scenarios/cross-version/source/**/*.cy.spec.js
+ - name: Stop Metabase ${{ matrix.version.source }}
+ run: docker stop metabase-${{ matrix.version.source }}
+
+ - name: Set target Docker image to OSS
+ run: |
+ echo "DOCKER_TARGET_IMAGE=metabase/metabase:${{ matrix.version.source }}" >> $GITHUB_ENV
+ if: startsWith(matrix.version.source, 'v0')
+ - name: Set target Docker image to EE
+ run: |
+ echo "DOCKER_TARGET_IMAGE=metabase/metabase-enterprise:${{ matrix.version.source }}" >> $GITHUB_ENV
+ if: startsWith(matrix.version.source, 'v1')
+ - name: Start Metabase ${{ matrix.version.target }}
+ run: |
+ docker run -d \
+ -v $PWD/my-metabase:/metabase.db \
+ -p 3001:3000 \
+ -e MB_PREMIUM_EMBEDDING_TOKEN \
+ --name metabase-${{ matrix.version.target }} \
+ ${{ env.DOCKER_TARGET_IMAGE }}
+ - name: Wait for Metabase to start on port 3001
+ run: while ! curl -s 'http://localhost:3001/api/health' | grep '{"status":"ok"}'; do sleep 1; done
+ - name: Run Cypress on the target
+ run: |
+ yarn cypress run \
+ --browser chrome \
+ --config-file frontend/test/metabase/scenarios/cross-version/target/cypress.json \
+ --spec frontend/test/metabase/scenarios/cross-version/target/**/*.cy.spec.js
+ - name: Upload Cypress Artifacts upon failure
+ uses: actions/upload-artifact@v2
+ if: failure()
+ with:
+ name: cypress-artifacts
+ path: |
+ ./cypress
+ if-no-files-found: ignore
diff --git a/frontend/test/__support__/e2e/cypress-plugins.js b/frontend/test/__support__/e2e/cypress-plugins.js
index 9b8a2c630ee..111e6e4937b 100644
--- a/frontend/test/__support__/e2e/cypress-plugins.js
+++ b/frontend/test/__support__/e2e/cypress-plugins.js
@@ -23,6 +23,9 @@ const snowplowMicroUrl = process.env["MB_SNOWPLOW_URL"];
const isQaDatabase = process.env["QA_DB_ENABLED"];
+const sourceVersion = process.env["CROSS_VERSION_SOURCE"];
+const targetVersion = process.env["CROSS_VERSION_TARGET"];
+
// This function is called when a project is opened or re-opened (e.g. due to
// the project's config changing)
@@ -72,6 +75,8 @@ module.exports = (on, config) => {
config.env.HAS_ENTERPRISE_TOKEN = hasEnterpriseToken;
config.env.HAS_SNOWPLOW_MICRO = hasSnowplowMicro;
config.env.SNOWPLOW_MICRO_URL = snowplowMicroUrl;
+ config.env.SOURCE_VERSION = sourceVersion;
+ config.env.TARGET_VERSION = targetVersion;
return config;
};
diff --git a/frontend/test/metabase/scenarios/cross-version/source/cross-version-source-helpers.js b/frontend/test/metabase/scenarios/cross-version/source/cross-version-source-helpers.js
new file mode 100644
index 00000000000..cdfc9f4eb35
--- /dev/null
+++ b/frontend/test/metabase/scenarios/cross-version/source/cross-version-source-helpers.js
@@ -0,0 +1,45 @@
+export const version = Cypress.env("SOURCE_VERSION");
+
+export function setupLanguage() {
+ // Make sure English is the default selected language
+ cy.findByText("English")
+ .should("have.css", "background-color")
+ .and("eq", "rgb(80, 158, 227)");
+
+ cy.button("Next").click();
+ cy.findByText("Your language is set to English");
+}
+
+export function setupInstance(version) {
+ const companyLabel =
+ version === "v0.41.3.1"
+ ? "Your company or team name"
+ : "Company or team name";
+
+ const finalSetupButton = version === "v0.41.3.1" ? "Next" : "Finish";
+
+ cy.findByLabelText("First name").type("Superuser");
+ cy.findByLabelText("Last name").type("Tableton");
+ cy.findByLabelText("Email").type("admin@metabase.test");
+ cy.findByLabelText(companyLabel).type("Metabase");
+ cy.findByLabelText("Create a password").type("12341234");
+ cy.findByLabelText("Confirm your password").type("12341234");
+ cy.button("Next").click();
+ cy.findByText("Hi, Superuser. Nice to meet you!");
+
+ cy.findByText("I'll add my data later").click();
+ cy.findByText("I'll add my own data later");
+
+ // Collection defaults to on and describes data collection
+ cy.findByText("All collection is completely anonymous.");
+ // turn collection off, which hides data collection description
+ cy.findByLabelText(
+ "Allow Metabase to anonymously collect usage events",
+ ).click();
+ cy.findByText("All collection is completely anonymous.").should("not.exist");
+ cy.findByText(finalSetupButton).click();
+ cy.findByText("Take me to Metabase").click();
+
+ cy.location("pathname").should("eq", "/");
+ cy.contains("Reviews");
+}
diff --git a/frontend/test/metabase/scenarios/cross-version/source/cypress.json b/frontend/test/metabase/scenarios/cross-version/source/cypress.json
new file mode 100644
index 00000000000..13053892f35
--- /dev/null
+++ b/frontend/test/metabase/scenarios/cross-version/source/cypress.json
@@ -0,0 +1,12 @@
+{
+ "testFiles": "**/*.cy.spec.js",
+ "integrationFolder": "frontend/test/metabase/scenarios/cross-version/source/",
+ "pluginsFile": "frontend/test/__support__/e2e/cypress-plugins.js",
+ "supportFile": "frontend/test/__support__/e2e/cypress.js",
+ "baseUrl": "http://localhost:3000",
+ "videoUploadOnPasses": false,
+ "chromeWebSecurity": false,
+ "modifyObstructiveCode": false,
+ "viewportHeight": 800,
+ "viewportWidth": 1280
+}
diff --git a/frontend/test/metabase/scenarios/cross-version/source/setup.cy.spec.js b/frontend/test/metabase/scenarios/cross-version/source/setup.cy.spec.js
new file mode 100644
index 00000000000..cb80c0801b4
--- /dev/null
+++ b/frontend/test/metabase/scenarios/cross-version/source/setup.cy.spec.js
@@ -0,0 +1,30 @@
+import {
+ version,
+ setupLanguage,
+ setupInstance,
+} from "./cross-version-source-helpers";
+
+describe(`setup on ${version}`, () => {
+ it("should set up metabase", () => {
+ cy.visit("/");
+ // It redirects to the setup page
+ cy.location("pathname").should("eq", "/setup");
+ cy.findByText("Welcome to Metabase");
+ cy.findByText("Let's get started").click();
+
+ setupLanguage();
+ setupInstance(version);
+
+ // Quick and dirty sanity check for EE version
+ // TODO: Remove or refactor properly
+ if (version.startsWith("v1")) {
+ cy.visit("/admin/settings/license");
+ cy.findByPlaceholderText("Using MB_PREMIUM_EMBEDDING_TOKEN").should(
+ "be.disabled",
+ );
+ } else {
+ cy.visit("/admin");
+ cy.icon("store");
+ }
+ });
+});
diff --git a/frontend/test/metabase/scenarios/cross-version/target/cross-version-target-helpers.js b/frontend/test/metabase/scenarios/cross-version/target/cross-version-target-helpers.js
new file mode 100644
index 00000000000..a6c1895a31b
--- /dev/null
+++ b/frontend/test/metabase/scenarios/cross-version/target/cross-version-target-helpers.js
@@ -0,0 +1 @@
+export const version = Cypress.env("TARGET_VERSION");
diff --git a/frontend/test/metabase/scenarios/cross-version/target/cypress.json b/frontend/test/metabase/scenarios/cross-version/target/cypress.json
new file mode 100644
index 00000000000..c4f7176f064
--- /dev/null
+++ b/frontend/test/metabase/scenarios/cross-version/target/cypress.json
@@ -0,0 +1,12 @@
+{
+ "testFiles": "**/*.cy.spec.js",
+ "integrationFolder": "frontend/test/metabase/scenarios/cross-version/target/",
+ "pluginsFile": "frontend/test/__support__/e2e/cypress-plugins.js",
+ "supportFile": "frontend/test/__support__/e2e/cypress.js",
+ "baseUrl": "http://localhost:3001",
+ "videoUploadOnPasses": false,
+ "chromeWebSecurity": false,
+ "modifyObstructiveCode": false,
+ "viewportHeight": 800,
+ "viewportWidth": 1280
+}
diff --git a/frontend/test/metabase/scenarios/cross-version/target/smoke.cy.spec.js b/frontend/test/metabase/scenarios/cross-version/target/smoke.cy.spec.js
new file mode 100644
index 00000000000..6de513556c8
--- /dev/null
+++ b/frontend/test/metabase/scenarios/cross-version/target/smoke.cy.spec.js
@@ -0,0 +1,8 @@
+import { version } from "./cross-version-target-helpers";
+
+describe(`smoke test the migration to the version ${version}`, () => {
+ it("should already be set up", () => {
+ cy.visit("/");
+ cy.findByText("Sign in to Metabase");
+ });
+});
--
GitLab