From 8d37c85eea1f80545df820f893a2811917173ba7 Mon Sep 17 00:00:00 2001
From: Nemanja Glumac <31325167+nemanjaglumac@users.noreply.github.com>
Date: Wed, 5 May 2021 18:29:48 +0200
Subject: [PATCH] Merge split e2e ci groups from release branch (cherry-pick
from `release-x.39.x`) (#15932)
* Allow multiple `--spec` arguments in Cypress `run` command (#15917)
* Split e2e CI groups (#15924)
* Add option to choose between source-folder and test-files in CCI
* Rearrange Cypress files
* Split e2e workflows
* Split native queries into a separate CI group
---
.circleci/config.yml | 94 ++++++++++++++++---
frontend/test/__runner__/run_cypress_tests.js | 48 +++++-----
.../{ => admin}/audit/ad-hoc.cy.spec.js | 0
.../{ => admin}/audit/auditing.cy.spec.js | 0
.../permissions-baseline.cy.spec.js} | 0
.../{question => native}/data_ref.cy.spec.js | 0
.../{question => native}/native.cy.spec.js | 0
.../native_subquery.cy.spec.js | 0
.../snippets/snippet-permissions.cy.spec.js | 0
.../{ => native}/snippets/snippets.cy.spec.js | 0
.../{ => onboarding}/auth/signin.cy.spec.js | 0
.../{ => onboarding}/auth/sso.cy.spec.js | 0
.../home/activity-page.cy.spec.js | 0
.../home/overworld.cy.spec.js | 0
.../{ => onboarding}/home/search.cy.spec.js | 0
.../reference/databases.cy.spec.js | 0
.../reference/metrics.cy.spec.js | 0
.../{auth => onboarding}/search.cy.spec.js | 0
.../{ => onboarding}/setup/setup.cy.spec.js | 0
.../setup/user_settings.cy.spec.js | 0
.../internal/question.cy.spec.js | 0
.../{ => sharing}/alert/alert.cy.spec.js | 0
.../{ => sharing}/alert/alert_auth.cy.spec.js | 0
.../alert/email_alert.cy.spec.js | 0
.../{dashboard => sharing}/embed.cy.spec.js | 0
.../{public => sharing}/public.cy.spec.js | 0
.../{pulse => sharing}/pulse.cy.spec.js | 0
.../subscriptions.cy.spec.js | 0
.../trendline.cy.spec.js | 0
29 files changed, 105 insertions(+), 37 deletions(-)
rename frontend/test/metabase/scenarios/{ => admin}/audit/ad-hoc.cy.spec.js (100%)
rename frontend/test/metabase/scenarios/{ => admin}/audit/auditing.cy.spec.js (100%)
rename frontend/test/metabase/scenarios/{permissions.cy.spec.js => collections/permissions-baseline.cy.spec.js} (100%)
rename frontend/test/metabase/scenarios/{question => native}/data_ref.cy.spec.js (100%)
rename frontend/test/metabase/scenarios/{question => native}/native.cy.spec.js (100%)
rename frontend/test/metabase/scenarios/{question => native}/native_subquery.cy.spec.js (100%)
rename frontend/test/metabase/scenarios/{ => native}/snippets/snippet-permissions.cy.spec.js (100%)
rename frontend/test/metabase/scenarios/{ => native}/snippets/snippets.cy.spec.js (100%)
rename frontend/test/metabase/scenarios/{ => onboarding}/auth/signin.cy.spec.js (100%)
rename frontend/test/metabase/scenarios/{ => onboarding}/auth/sso.cy.spec.js (100%)
rename frontend/test/metabase/scenarios/{ => onboarding}/home/activity-page.cy.spec.js (100%)
rename frontend/test/metabase/scenarios/{ => onboarding}/home/overworld.cy.spec.js (100%)
rename frontend/test/metabase/scenarios/{ => onboarding}/home/search.cy.spec.js (100%)
rename frontend/test/metabase/scenarios/{ => onboarding}/reference/databases.cy.spec.js (100%)
rename frontend/test/metabase/scenarios/{ => onboarding}/reference/metrics.cy.spec.js (100%)
rename frontend/test/metabase/scenarios/{auth => onboarding}/search.cy.spec.js (100%)
rename frontend/test/metabase/scenarios/{ => onboarding}/setup/setup.cy.spec.js (100%)
rename frontend/test/metabase/scenarios/{ => onboarding}/setup/user_settings.cy.spec.js (100%)
rename frontend/test/metabase/scenarios/{ => question}/internal/question.cy.spec.js (100%)
rename frontend/test/metabase/scenarios/{ => sharing}/alert/alert.cy.spec.js (100%)
rename frontend/test/metabase/scenarios/{ => sharing}/alert/alert_auth.cy.spec.js (100%)
rename frontend/test/metabase/scenarios/{ => sharing}/alert/email_alert.cy.spec.js (100%)
rename frontend/test/metabase/scenarios/{dashboard => sharing}/embed.cy.spec.js (100%)
rename frontend/test/metabase/scenarios/{public => sharing}/public.cy.spec.js (100%)
rename frontend/test/metabase/scenarios/{pulse => sharing}/pulse.cy.spec.js (100%)
rename frontend/test/metabase/scenarios/{dashboard => sharing}/subscriptions.cy.spec.js (100%)
rename frontend/test/metabase/scenarios/{question => visualizations}/trendline.cy.spec.js (100%)
diff --git a/.circleci/config.yml b/.circleci/config.yml
index 70b91eac36c..ef419db159a 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -861,10 +861,10 @@ jobs:
default: clojure-and-node-and-browsers
cypress-group:
type: string
- only-single-database:
- type: boolean
- default: false
- test-files-location:
+ source-folder:
+ type: string
+ default: ""
+ test-files:
type: string
default: ""
before-steps:
@@ -889,7 +889,7 @@ jobs:
<<: *CacheKeyUberjar
- steps: << parameters.before-steps >>
command: |
- run test-cypress-no-build <<# parameters.only-single-database >> --folder << parameters.test-files-location >> <</ parameters.only-single-database >>
+ run test-cypress-no-build <<# parameters.test-files >> --spec << parameters.test-files >> <</ parameters.test-files >> --folder << parameters.source-folder >>
after-steps:
- store_artifacts:
path: /home/circleci/metabase/metabase/cypress
@@ -1156,10 +1156,81 @@ workflows:
matrix:
parameters:
edition: ["ee", "oss"]
- name: e2e-tests-<< matrix.edition >>
+ name: e2e-tests-admin-<< matrix.edition >>
+ requires:
+ - build-uberjar-<< matrix.edition >>
+ cypress-group: "admin-<< matrix.edition >>"
+ source-folder: frontend/test/metabase/scenarios/admin
+
+ - fe-tests-cypress:
+ matrix:
+ parameters:
+ edition: ["ee", "oss"]
+ name: e2e-tests-collections-<< matrix.edition >>
+ requires:
+ - build-uberjar-<< matrix.edition >>
+ cypress-group: "collections-<< matrix.edition >>"
+ source-folder: frontend/test/metabase/scenarios/collections
+
+ - fe-tests-cypress:
+ matrix:
+ parameters:
+ edition: ["ee", "oss"]
+ name: e2e-tests-dashboard-<< matrix.edition >>
+ requires:
+ - build-uberjar-<< matrix.edition >>
+ cypress-group: "dashboard-<< matrix.edition >>"
+ source-folder: frontend/test/metabase/scenarios/dashboard
+
+ - fe-tests-cypress:
+ matrix:
+ parameters:
+ edition: ["ee", "oss"]
+ name: e2e-tests-onboarding-<< matrix.edition >>
+ requires:
+ - build-uberjar-<< matrix.edition >>
+ cypress-group: "onboarding-<< matrix.edition >>"
+ source-folder: frontend/test/metabase/scenarios/onboarding
+
+ - fe-tests-cypress:
+ matrix:
+ parameters:
+ edition: ["ee", "oss"]
+ name: e2e-tests-native-<< matrix.edition >>
+ requires:
+ - build-uberjar-<< matrix.edition >>
+ cypress-group: "native-<< matrix.edition >>"
+ source-folder: frontend/test/metabase/scenarios/native
+
+ - fe-tests-cypress:
+ matrix:
+ parameters:
+ edition: ["ee", "oss"]
+ name: e2e-tests-question-<< matrix.edition >>
+ requires:
+ - build-uberjar-<< matrix.edition >>
+ cypress-group: "question-<< matrix.edition >>"
+ source-folder: frontend/test/metabase/scenarios/question
+
+ - fe-tests-cypress:
+ matrix:
+ parameters:
+ edition: ["ee", "oss"]
+ name: e2e-tests-sharing-<< matrix.edition >>
+ requires:
+ - build-uberjar-<< matrix.edition >>
+ cypress-group: "sharing-<< matrix.edition >>"
+ source-folder: frontend/test/metabase/scenarios/sharing
+
+ - fe-tests-cypress:
+ matrix:
+ parameters:
+ edition: ["ee", "oss"]
+ name: e2e-tests-visualizations-<< matrix.edition >>
requires:
- build-uberjar-<< matrix.edition >>
- cypress-group: "default-<< matrix.edition >>"
+ cypress-group: "visualizations-<< matrix.edition >>"
+ source-folder: frontend/test/metabase/scenarios/visualizations
- fe-tests-cypress:
name: e2e-tests-mongo-4-<< matrix.edition >>
@@ -1167,8 +1238,7 @@ workflows:
- build-uberjar-<< matrix.edition >>
e: fe-mongo-4
cypress-group: "mongo"
- only-single-database: true
- test-files-location: frontend/test/metabase-db/mongo
+ source-folder: frontend/test/metabase-db/mongo
before-steps:
- wait-for-port:
port: 27017
@@ -1180,8 +1250,7 @@ workflows:
- build-uberjar-<< matrix.edition >>
e: fe-postgres-12
cypress-group: "postgres"
- only-single-database: true
- test-files-location: frontend/test/metabase-db/postgres
+ source-folder: frontend/test/metabase-db/postgres
before-steps:
- wait-for-port:
port: 5432
@@ -1193,8 +1262,7 @@ workflows:
- build-uberjar-<< matrix.edition >>
e: fe-mysql-8
cypress-group: "mysql"
- only-single-database: true
- test-files-location: frontend/test/metabase-db/mysql
+ source-folder: frontend/test/metabase-db/mysql
before-steps:
- wait-for-port:
port: 3306
diff --git a/frontend/test/__runner__/run_cypress_tests.js b/frontend/test/__runner__/run_cypress_tests.js
index 9cb9f0149e2..64c8bc844fb 100644
--- a/frontend/test/__runner__/run_cypress_tests.js
+++ b/frontend/test/__runner__/run_cypress_tests.js
@@ -13,10 +13,12 @@ const server = BackendResource.get({ dbKey: "" });
// --spec <single-spec-path> - Specifies a path to a single test file
const userArgs = process.argv.slice(2);
const isOpenMode = userArgs.includes("--open");
-const sourceFolder = userArgs.includes("--folder");
-const singleFile = userArgs.includes("--spec");
+const isFolderFlag = userArgs.includes("--folder");
+const isSpecFlag = userArgs.includes("--spec");
const sourceFolderLocation = userArgs[userArgs.indexOf("--folder") + 1];
-const singleFileName = userArgs[userArgs.indexOf("--spec") + 1];
+const specs = userArgs[userArgs.indexOf("--spec") + 1];
+const isSingleSpec = !specs.match(/,/);
+const testFiles = isSingleSpec ? specs : specs.split(",");
function readFile(fileName) {
return new Promise(function(resolve, reject) {
@@ -38,46 +40,40 @@ const init = async () => {
);
}
- if (sourceFolder) {
- console.log(chalk.bold(`Running tests in '${sourceFolderLocation}'`));
- }
+ const logMessage = isFolderFlag
+ ? `Running tests in '${sourceFolderLocation}'`
+ : `Running '${testFiles}'`;
- if (singleFile) {
- console.log(chalk.bold(`Running single spec '${singleFileName}'`));
- }
+ printBold(logMessage);
try {
const version = await readFile(
__dirname + "/../../../resources/version.properties",
);
- console.log(chalk.bold("Running e2e test runner with this build:"));
+ printBold("Running e2e test runner with this build:");
process.stdout.write(chalk.cyan(version));
- console.log(
- chalk.bold(
- "If that version seems too old, please run `./bin/build version uberjar`.\n",
- ),
+ printBold(
+ "If that version seems too old, please run `./bin/build version uberjar`.\n",
);
} catch (e) {
- console.log(
- chalk.bold(
- "No version file found. Please run `./bin/build version uberjar`.",
- ),
+ printBold(
+ "No version file found. Please run `./bin/build version uberjar`.",
);
process.exit(1);
}
- console.log(chalk.bold("Starting backend"));
+ printBold("Starting backend");
await BackendResource.start(server);
- console.log(chalk.bold("Generating snapshots"));
+ printBold("Generating snapshots");
await generateSnapshots();
- console.log(chalk.bold("Starting Cypress"));
+ printBold("Starting Cypress");
const baseConfig = { baseUrl: server.host };
- const folderConfig = sourceFolder && {
+ const folderConfig = isFolderFlag && {
integrationFolder: sourceFolderLocation,
};
- const specsConfig = singleFile && { testFiles: singleFileName };
+ const specsConfig = isSpecFlag && { testFiles };
const ignoreConfig =
// if we're not running specific tests, avoid including db and smoketests
folderConfig || specsConfig
@@ -127,7 +123,7 @@ const init = async () => {
};
const cleanup = async (exitCode = 0) => {
- console.log(chalk.bold("Cleaning up..."));
+ printBold("Cleaning up...");
await BackendResource.stop(server);
process.exit(exitCode);
};
@@ -163,3 +159,7 @@ async function generateSnapshots() {
cypressProcess.on("exit", resolve);
});
}
+
+function printBold(message) {
+ console.log(chalk.bold(message));
+}
diff --git a/frontend/test/metabase/scenarios/audit/ad-hoc.cy.spec.js b/frontend/test/metabase/scenarios/admin/audit/ad-hoc.cy.spec.js
similarity index 100%
rename from frontend/test/metabase/scenarios/audit/ad-hoc.cy.spec.js
rename to frontend/test/metabase/scenarios/admin/audit/ad-hoc.cy.spec.js
diff --git a/frontend/test/metabase/scenarios/audit/auditing.cy.spec.js b/frontend/test/metabase/scenarios/admin/audit/auditing.cy.spec.js
similarity index 100%
rename from frontend/test/metabase/scenarios/audit/auditing.cy.spec.js
rename to frontend/test/metabase/scenarios/admin/audit/auditing.cy.spec.js
diff --git a/frontend/test/metabase/scenarios/permissions.cy.spec.js b/frontend/test/metabase/scenarios/collections/permissions-baseline.cy.spec.js
similarity index 100%
rename from frontend/test/metabase/scenarios/permissions.cy.spec.js
rename to frontend/test/metabase/scenarios/collections/permissions-baseline.cy.spec.js
diff --git a/frontend/test/metabase/scenarios/question/data_ref.cy.spec.js b/frontend/test/metabase/scenarios/native/data_ref.cy.spec.js
similarity index 100%
rename from frontend/test/metabase/scenarios/question/data_ref.cy.spec.js
rename to frontend/test/metabase/scenarios/native/data_ref.cy.spec.js
diff --git a/frontend/test/metabase/scenarios/question/native.cy.spec.js b/frontend/test/metabase/scenarios/native/native.cy.spec.js
similarity index 100%
rename from frontend/test/metabase/scenarios/question/native.cy.spec.js
rename to frontend/test/metabase/scenarios/native/native.cy.spec.js
diff --git a/frontend/test/metabase/scenarios/question/native_subquery.cy.spec.js b/frontend/test/metabase/scenarios/native/native_subquery.cy.spec.js
similarity index 100%
rename from frontend/test/metabase/scenarios/question/native_subquery.cy.spec.js
rename to frontend/test/metabase/scenarios/native/native_subquery.cy.spec.js
diff --git a/frontend/test/metabase/scenarios/snippets/snippet-permissions.cy.spec.js b/frontend/test/metabase/scenarios/native/snippets/snippet-permissions.cy.spec.js
similarity index 100%
rename from frontend/test/metabase/scenarios/snippets/snippet-permissions.cy.spec.js
rename to frontend/test/metabase/scenarios/native/snippets/snippet-permissions.cy.spec.js
diff --git a/frontend/test/metabase/scenarios/snippets/snippets.cy.spec.js b/frontend/test/metabase/scenarios/native/snippets/snippets.cy.spec.js
similarity index 100%
rename from frontend/test/metabase/scenarios/snippets/snippets.cy.spec.js
rename to frontend/test/metabase/scenarios/native/snippets/snippets.cy.spec.js
diff --git a/frontend/test/metabase/scenarios/auth/signin.cy.spec.js b/frontend/test/metabase/scenarios/onboarding/auth/signin.cy.spec.js
similarity index 100%
rename from frontend/test/metabase/scenarios/auth/signin.cy.spec.js
rename to frontend/test/metabase/scenarios/onboarding/auth/signin.cy.spec.js
diff --git a/frontend/test/metabase/scenarios/auth/sso.cy.spec.js b/frontend/test/metabase/scenarios/onboarding/auth/sso.cy.spec.js
similarity index 100%
rename from frontend/test/metabase/scenarios/auth/sso.cy.spec.js
rename to frontend/test/metabase/scenarios/onboarding/auth/sso.cy.spec.js
diff --git a/frontend/test/metabase/scenarios/home/activity-page.cy.spec.js b/frontend/test/metabase/scenarios/onboarding/home/activity-page.cy.spec.js
similarity index 100%
rename from frontend/test/metabase/scenarios/home/activity-page.cy.spec.js
rename to frontend/test/metabase/scenarios/onboarding/home/activity-page.cy.spec.js
diff --git a/frontend/test/metabase/scenarios/home/overworld.cy.spec.js b/frontend/test/metabase/scenarios/onboarding/home/overworld.cy.spec.js
similarity index 100%
rename from frontend/test/metabase/scenarios/home/overworld.cy.spec.js
rename to frontend/test/metabase/scenarios/onboarding/home/overworld.cy.spec.js
diff --git a/frontend/test/metabase/scenarios/home/search.cy.spec.js b/frontend/test/metabase/scenarios/onboarding/home/search.cy.spec.js
similarity index 100%
rename from frontend/test/metabase/scenarios/home/search.cy.spec.js
rename to frontend/test/metabase/scenarios/onboarding/home/search.cy.spec.js
diff --git a/frontend/test/metabase/scenarios/reference/databases.cy.spec.js b/frontend/test/metabase/scenarios/onboarding/reference/databases.cy.spec.js
similarity index 100%
rename from frontend/test/metabase/scenarios/reference/databases.cy.spec.js
rename to frontend/test/metabase/scenarios/onboarding/reference/databases.cy.spec.js
diff --git a/frontend/test/metabase/scenarios/reference/metrics.cy.spec.js b/frontend/test/metabase/scenarios/onboarding/reference/metrics.cy.spec.js
similarity index 100%
rename from frontend/test/metabase/scenarios/reference/metrics.cy.spec.js
rename to frontend/test/metabase/scenarios/onboarding/reference/metrics.cy.spec.js
diff --git a/frontend/test/metabase/scenarios/auth/search.cy.spec.js b/frontend/test/metabase/scenarios/onboarding/search.cy.spec.js
similarity index 100%
rename from frontend/test/metabase/scenarios/auth/search.cy.spec.js
rename to frontend/test/metabase/scenarios/onboarding/search.cy.spec.js
diff --git a/frontend/test/metabase/scenarios/setup/setup.cy.spec.js b/frontend/test/metabase/scenarios/onboarding/setup/setup.cy.spec.js
similarity index 100%
rename from frontend/test/metabase/scenarios/setup/setup.cy.spec.js
rename to frontend/test/metabase/scenarios/onboarding/setup/setup.cy.spec.js
diff --git a/frontend/test/metabase/scenarios/setup/user_settings.cy.spec.js b/frontend/test/metabase/scenarios/onboarding/setup/user_settings.cy.spec.js
similarity index 100%
rename from frontend/test/metabase/scenarios/setup/user_settings.cy.spec.js
rename to frontend/test/metabase/scenarios/onboarding/setup/user_settings.cy.spec.js
diff --git a/frontend/test/metabase/scenarios/internal/question.cy.spec.js b/frontend/test/metabase/scenarios/question/internal/question.cy.spec.js
similarity index 100%
rename from frontend/test/metabase/scenarios/internal/question.cy.spec.js
rename to frontend/test/metabase/scenarios/question/internal/question.cy.spec.js
diff --git a/frontend/test/metabase/scenarios/alert/alert.cy.spec.js b/frontend/test/metabase/scenarios/sharing/alert/alert.cy.spec.js
similarity index 100%
rename from frontend/test/metabase/scenarios/alert/alert.cy.spec.js
rename to frontend/test/metabase/scenarios/sharing/alert/alert.cy.spec.js
diff --git a/frontend/test/metabase/scenarios/alert/alert_auth.cy.spec.js b/frontend/test/metabase/scenarios/sharing/alert/alert_auth.cy.spec.js
similarity index 100%
rename from frontend/test/metabase/scenarios/alert/alert_auth.cy.spec.js
rename to frontend/test/metabase/scenarios/sharing/alert/alert_auth.cy.spec.js
diff --git a/frontend/test/metabase/scenarios/alert/email_alert.cy.spec.js b/frontend/test/metabase/scenarios/sharing/alert/email_alert.cy.spec.js
similarity index 100%
rename from frontend/test/metabase/scenarios/alert/email_alert.cy.spec.js
rename to frontend/test/metabase/scenarios/sharing/alert/email_alert.cy.spec.js
diff --git a/frontend/test/metabase/scenarios/dashboard/embed.cy.spec.js b/frontend/test/metabase/scenarios/sharing/embed.cy.spec.js
similarity index 100%
rename from frontend/test/metabase/scenarios/dashboard/embed.cy.spec.js
rename to frontend/test/metabase/scenarios/sharing/embed.cy.spec.js
diff --git a/frontend/test/metabase/scenarios/public/public.cy.spec.js b/frontend/test/metabase/scenarios/sharing/public.cy.spec.js
similarity index 100%
rename from frontend/test/metabase/scenarios/public/public.cy.spec.js
rename to frontend/test/metabase/scenarios/sharing/public.cy.spec.js
diff --git a/frontend/test/metabase/scenarios/pulse/pulse.cy.spec.js b/frontend/test/metabase/scenarios/sharing/pulse.cy.spec.js
similarity index 100%
rename from frontend/test/metabase/scenarios/pulse/pulse.cy.spec.js
rename to frontend/test/metabase/scenarios/sharing/pulse.cy.spec.js
diff --git a/frontend/test/metabase/scenarios/dashboard/subscriptions.cy.spec.js b/frontend/test/metabase/scenarios/sharing/subscriptions.cy.spec.js
similarity index 100%
rename from frontend/test/metabase/scenarios/dashboard/subscriptions.cy.spec.js
rename to frontend/test/metabase/scenarios/sharing/subscriptions.cy.spec.js
diff --git a/frontend/test/metabase/scenarios/question/trendline.cy.spec.js b/frontend/test/metabase/scenarios/visualizations/trendline.cy.spec.js
similarity index 100%
rename from frontend/test/metabase/scenarios/question/trendline.cy.spec.js
rename to frontend/test/metabase/scenarios/visualizations/trendline.cy.spec.js
--
GitLab