[Cypress runner] Run Cypress (#17099)

Related issue: https://github.com/metabase/metabase/issues/17005

* Add `arg` library

* Rewrite the main Cypress runner logic

* Destructure Cypress snapshot results

* Rename Cypress snapshots runner

* Update CircleCI config to reflect `folder` changes

* Update readme for smoke tests

* Run `metabase-db` tests conditionally based on env `QA_DB_ENABLED`

.circleci/config.yml

@@ -883,6 +883,9 @@ jobs:
       test-files:
         type: string
         default: ""
+      qa-db:
+        type: boolean
+        default: false
       before-steps:
         type: steps
         default: []
@@ -891,6 +894,7 @@ jobs:
     environment:
       MB_EDITION: << parameters.edition >>
       CYPRESS_GROUP:  << parameters.cypress-group >>
+      QA_DB_ENABLED: << parameters.qa-db >>
       DISPLAY: ""
     steps:
       - attach-workspace
@@ -1177,7 +1181,7 @@ workflows:
           requires:
             - build-uberjar-<< matrix.edition >>
           cypress-group: "smoketest-<< matrix.edition >>"
-          source-folder: frontend/test/metabase/scenarios/smoketest
+          source-folder: smoketest
 
       - fe-tests-cypress:
           matrix:
@@ -1187,7 +1191,7 @@ workflows:
           requires:
             - build-uberjar-<< matrix.edition >>
           cypress-group: "admin-<< matrix.edition >>"
-          source-folder: frontend/test/metabase/scenarios/admin
+          source-folder: admin
 
       - fe-tests-cypress:
           matrix:
@@ -1197,7 +1201,7 @@ workflows:
           requires:
             - build-uberjar-<< matrix.edition >>
           cypress-group: "collections-<< matrix.edition >>"
-          source-folder: frontend/test/metabase/scenarios/collections
+          source-folder: collections
 
       - fe-tests-cypress:
           matrix:
@@ -1207,7 +1211,7 @@ workflows:
           requires:
             - build-uberjar-<< matrix.edition >>
           cypress-group: "dashboard-<< matrix.edition >>"
-          source-folder: frontend/test/metabase/scenarios/dashboard
+          source-folder: dashboard
 
       - fe-tests-cypress:
           matrix:
@@ -1217,7 +1221,7 @@ workflows:
           requires:
             - build-uberjar-<< matrix.edition >>
           cypress-group: "dashboard-filters-<< matrix.edition >>"
-          source-folder: frontend/test/metabase/scenarios/dashboard-filters
+          source-folder: dashboard-filters
 
       - fe-tests-cypress:
           matrix:
@@ -1227,7 +1231,7 @@ workflows:
           requires:
             - build-uberjar-<< matrix.edition >>
           cypress-group: "onboarding-<< matrix.edition >>"
-          source-folder: frontend/test/metabase/scenarios/onboarding
+          source-folder: onboarding
 
       - fe-tests-cypress:
           matrix:
@@ -1237,7 +1241,7 @@ workflows:
           requires:
             - build-uberjar-<< matrix.edition >>
           cypress-group: "native-<< matrix.edition >>"
-          source-folder: frontend/test/metabase/scenarios/native
+          source-folder: native
 
       - fe-tests-cypress:
           matrix:
@@ -1247,7 +1251,7 @@ workflows:
           requires:
             - build-uberjar-<< matrix.edition >>
           cypress-group: "native-filters-<< matrix.edition >>"
-          source-folder: frontend/test/metabase/scenarios/native-filters
+          source-folder: native-filters
 
       - fe-tests-cypress:
           matrix:
@@ -1257,7 +1261,7 @@ workflows:
           requires:
             - build-uberjar-<< matrix.edition >>
           cypress-group: "question-<< matrix.edition >>"
-          source-folder: frontend/test/metabase/scenarios/question
+          source-folder: question
 
       - fe-tests-cypress:
           matrix:
@@ -1267,7 +1271,7 @@ workflows:
           requires:
             - build-uberjar-<< matrix.edition >>
           cypress-group: "binning-<< matrix.edition >>"
-          source-folder: frontend/test/metabase/scenarios/binning
+          source-folder: binning
 
       - fe-tests-cypress:
           matrix:
@@ -1277,7 +1281,7 @@ workflows:
           requires:
             - build-uberjar-<< matrix.edition >>
           cypress-group: "sharing-<< matrix.edition >>"
-          source-folder: frontend/test/metabase/scenarios/sharing
+          source-folder: sharing
 
       - fe-tests-cypress:
           matrix:
@@ -1287,7 +1291,7 @@ workflows:
           requires:
             - build-uberjar-<< matrix.edition >>
           cypress-group: "visualizations-<< matrix.edition >>"
-          source-folder: frontend/test/metabase/scenarios/visualizations
+          source-folder: visualizations
 
       - fe-tests-cypress:
           name: e2e-tests-mongo-4-<< matrix.edition >>
@@ -1295,7 +1299,8 @@ workflows:
             - build-uberjar-<< matrix.edition >>
           e: fe-mongo-4
           cypress-group: "mongo"
-          source-folder: frontend/test/metabase-db/mongo
+          source-folder: mongo
+          qa-db: true
           before-steps:
             - wait-for-port:
                 port: 27017
@@ -1307,7 +1312,8 @@ workflows:
             - build-uberjar-<< matrix.edition >>
           e: fe-postgres-12
           cypress-group: "postgres"
-          source-folder: frontend/test/metabase-db/postgres
+          source-folder: postgres
+          qa-db: true
           before-steps:
             - wait-for-port:
                 port: 5432
@@ -1319,7 +1325,8 @@ workflows:
             - build-uberjar-<< matrix.edition >>
           e: fe-mysql-8
           cypress-group: "mysql"
-          source-folder: frontend/test/metabase-db/mysql
+          source-folder: mysql
+          qa-db: true
           before-steps:
             - wait-for-port:
                 port: 3306

frontend/test/__runner__/generate-cypress-snapshots.js → frontend/test/__runner__/cypress-runner-generate-snapshots.js

@@ -13,22 +13,24 @@ const generateSnapshots = async (baseUrl, exitFunction) => {
   const snapshotConfig = getConfig(baseUrl);
 
   try {
-    const results = await cypress.run(snapshotConfig);
+    const { status, message, totalFailed, failures } = await cypress.run(
+      snapshotConfig,
+    );
 
     // At least one test failed. We can't continue to the next step.
     // Cypress tests rely on snapshots correctly generated at this stage.
-    if (results.totalFailed > 0) {
+    if (totalFailed > 0) {
       await exitFunction(1);
     }
 
     // Something went wrong and Cypress failed to even run tests
-    if (results.status === "failed" && results.failures) {
-      console.error(results.message);
+    if (status === "failed" && failures) {
+      console.error(message);
 
-      await exitFunction(results.failures);
+      await exitFunction(failures);
     }
   } catch (e) {
-    console.error("Unable to generate snapshots\n", e);
+    console.error("Unable to generate snapshots!\n", e);
 
     await exitFunction(1);
   }

frontend/test/__runner__/cypress-runner-run-tests.js

+const cypress = require("cypress");
+const arg = require("arg");
+
+const args = arg(
+  {
+    "--folder": String, // The name of the folder to run files from
+    "--open": [Boolean], // Run Cypress in open mode or not? Doesn't accept additional arguments
+  },
+  { permissive: true }, // Passes all other flags and args to the Cypress parser
+);
+
+const folder = args["--folder"];
+const isFolder = !!folder;
+
+const supportedDatabases = ["mongo", "mysql", "postgres"];
+const isQaDatabase = supportedDatabases.includes(folder);
+
+const isOpenMode = args["--open"];
+const isCI = process.env["CI"];
+
+const parseArguments = async () => {
+  const cliArgs = args._;
+
+  // cypress.cli.parseArguments requires `cypress run` as the first two arguments
+  if (cliArgs[0] !== "cypress") {
+    cliArgs.unshift("cypress");
+  }
+
+  if (cliArgs[1] !== "run") {
+    cliArgs.splice(1, 0, "run");
+  }
+
+  return await cypress.cli.parseRunArguments(cliArgs);
+};
+
+// This overly complicated logic will not be needed once we merge "db" specs with the rest of the "normal" Cypress files.
+// Alternatively we can use the official `--project` flag to isolate "db" files but that would require a major refactor.
+const getSourceFolder = () => {
+  const defaultPath = `./frontend/test/metabase/scenarios/${folder}/**/*.cy.spec.js`;
+  const QaDatabasePath = `./frontend/test/metabase-db/${folder}/**/*.cy.spec.js`;
+
+  return isQaDatabase ? QaDatabasePath : defaultPath;
+};
+
+const getReporterConfig = isCI => {
+  return isCI
+    ? {
+        reporter: "junit",
+        "reporter-options": "mochaFile=cypress/results/results-[hash].xml",
+      }
+    : null;
+};
+
+const runCypress = async (baseUrl, exitFunction) => {
+  const defaultConfig = {
+    configFile: "frontend/test/__support__/e2e/cypress.json",
+    config: {
+      baseUrl,
+    },
+    spec: isFolder && getSourceFolder(),
+  };
+
+  const reporterConfig = getReporterConfig(isCI);
+
+  const userArgs = await parseArguments();
+
+  const finalConfig = Object.assign(
+    {},
+    defaultConfig,
+    reporterConfig,
+    userArgs,
+  );
+
+  try {
+    const { status, message, totalFailed, failures } = isOpenMode
+      ? await cypress.open(finalConfig)
+      : await cypress.run(finalConfig);
+
+    // At least one test failed
+    if (totalFailed > 0) {
+      await exitFunction(1);
+    }
+
+    // Something went wrong and Cypress failed to even run tests
+    if (status === "failed" && failures) {
+      console.error(message);
+
+      await exitFunction(failures);
+    }
+  } catch (e) {
+    console.error("Failed to run Cypress!\n", e);
+
+    await exitFunction(1);
+  }
+};
+
+module.exports = runCypress;

frontend/test/__runner__/run_cypress_tests.js

-import { spawn } from "child_process";
-
+const { printBold } = require("./cypress-runner-utils");
+const runCypress = require("./cypress-runner-run-tests");
 const getVersion = require("./cypress-runner-get-version");
-const generateSnapshots = require("./generate-cypress-snapshots");
-const { printBold, printYellow } = require("./cypress-runner-utils");
+const generateSnapshots = require("./cypress-runner-generate-snapshots");
 
 // Use require for BackendResource to run it after the mock afterAll has been set
 const BackendResource = require("./backend.js").BackendResource;
@@ -10,19 +9,6 @@ const BackendResource = require("./backend.js").BackendResource;
 const server = BackendResource.get({ dbKey: "" });
 const baseUrl = server.host;
 
-// We currently accept three (optional) command line arguments
-// --open - Opens the Cypress test browser
-// --folder <path> - Specifies a different path for the integration folder
-// --spec <single-spec-path> - Specifies a path to a single test file
-const userArgs = process.argv.slice(2);
-const isOpenMode = userArgs.includes("--open");
-const isFolderFlag = userArgs.includes("--folder");
-const isSpecFlag = userArgs.includes("--spec");
-const sourceFolderLocation = userArgs[userArgs.indexOf("--folder") + 1];
-const specs = userArgs[userArgs.indexOf("--spec") + 1];
-const isSingleSpec = !specs || !specs.match(/,/);
-const testFiles = isSingleSpec ? specs : specs.split(",");
-
 const init = async () => {
   printBold("Metabase version info");
   await getVersion();
@@ -34,62 +20,7 @@ const init = async () => {
   await generateSnapshots(baseUrl, cleanup);
 
   printBold("Starting Cypress");
-  if (!isOpenMode) {
-    printYellow(
-      "If you are developing locally, prefer using `yarn test-cypress-open` instead.\n",
-    );
-  }
-
-  const logMessage = isFolderFlag
-    ? `Running tests in '${sourceFolderLocation}'`
-    : `Running '${testFiles}'`;
-
-  printBold(logMessage);
-  const baseConfig = { baseUrl: server.host };
-  const folderConfig = isFolderFlag && {
-    integrationFolder: sourceFolderLocation,
-  };
-  const specsConfig = isSpecFlag && { testFiles };
-  const ignoreConfig =
-    // if we're not running specific tests, avoid including db tests
-    folderConfig || specsConfig
-      ? null
-      : { ignoreTestFiles: "**/metabase-db/**" };
-
-  const config = {
-    ...baseConfig,
-    ...folderConfig,
-    ...specsConfig,
-    ...ignoreConfig,
-  };
-  // Cypress suggests using JSON.stringified object for more complex configuration objects
-  // See: https://docs.cypress.io/guides/references/configuration#Command-Line
-  const commandLineConfig = JSON.stringify(config);
-
-  const cypressProcess = spawn(
-    "yarn",
-    [
-      "cypress",
-      isOpenMode ? "open" : "run",
-      "--config-file",
-      process.env["CONFIG_FILE"],
-      "--config",
-      commandLineConfig,
-      ...(process.env["CI"]
-        ? [
-            "--reporter",
-            "junit",
-            "--reporter-options",
-            "mochaFile=cypress/results/results-[hash].xml",
-          ]
-        : []),
-    ],
-    { stdio: "inherit" },
-  );
-
-  return new Promise((resolve, reject) => {
-    cypressProcess.on("exit", resolve);
-  });
+  await runCypress(baseUrl, cleanup);
 };
 
 const cleanup = async (exitCode = 0) => {

frontend/test/__support__/e2e/cypress-plugins.js

@@ -17,6 +17,8 @@
 const hasEnterpriseToken =
   process.env["ENTERPRISE_TOKEN"] && process.env["MB_EDITION"] === "ee";
 
+const isQaDatabase = process.env["QA_DB_ENABLED"];
+
 // This function is called when a project is opened or re-opened (e.g. due to
 // the project's config changing)
 const webpack = require("@cypress/webpack-preprocessor");
@@ -54,6 +56,9 @@ module.exports = (on, config) => {
    **                          CONFIG                                **
    ********************************************************************/
 
+  if (!isQaDatabase) {
+    config.ignoreTestFiles = "**/metabase-db/**";
+  }
   config.env.HAS_ENTERPRISE_TOKEN = hasEnterpriseToken;
 
   return config;

frontend/test/metabase-db/README.md

+# Metabase QA DB Tests
+
+These files ensure that Metabase is working with some of the supported databases.
+
+## Running
+
+From the root of the repository:
+
+- If you already have built Metabase with `./bin/build`, just run this to run all the tests.
+
+```shell
+#Choose one
+yarn run test-cypress-no-build --folder mongo|mysql|postgres
+
+# or run all tests
+yarn run test-cypress-no-build --spec ./frontend/test/metabase-db/**/*.cy.spec.js
+```
+
+- Active development, add `--open`
+
+### Requirements
+
+Prior to running these tests, please make sure tha you have enabled the `QA_DB_ENABLED` env for that session only.
+
+```bash
+QA_DB_ENABLED=true
+```
+
+```fish
+set -x QA_Db_ENABLED true
+```
+
+The list of all supported databases that we currently have Docker images for can be found at [Metabase QA repo](https://github.com/metabase/metabase-qa).
+
+For the convenience, we'll list docker commands here as well:
+
+```shell
+#Mongo 4
+docker run --rm -p 27017:27017 --name meta-mongo-sample metabase/qa-databases:mongo-sample-4.0
+
+#PostgreSQL 12
+docker run --rm -p 5432:5432 --name meta-postgres12-sample metabase/qa-databases:postgres-sample-12
+
+#MySQL 8
+docker run --rm -p 3306:3306 --name meta-mysql8-sample metabase/qa-databases:mysql-sample-8
+```

frontend/test/metabase/scenarios/smoketest/README.md

@@ -6,12 +6,10 @@ These files exist to ensure that we can always do the things we expect with Meta
 
 From the root of the repository:
 
-- If you are running tests that include admin_setup, run `python -m smtpd -n -c DebuggingServer localhost:1025`in terminal first for setting up email through your localhost
-
 - If you already have built Metabase with `./bin/build`, just run this to run all the tests.
 
 ```shell
-yarn run test-cypress-no-build --folder frontend/test/metabase/scenarios/smoketest
+yarn run test-cypress-no-build --folder smoketest
 ```
 
 - Active development, add `--open`

package.json

@@ -11,6 +11,7 @@
   },
   "dependencies": {
     "ace-builds": "^1.4.7",
+    "arg": "^5.0.0",
     "chevrotain": "^6.5.0",
     "classlist-polyfill": "^1.2.0",
     "classnames": "^2.1.3",
@@ -201,7 +202,7 @@
     "test-cypress": "yarn build && ./bin/build-for-test && yarn test-cypress-no-build",
     "test-cypress-open": "./bin/build-for-test && yarn test-cypress-no-build --open",
     "test-cypress-open-no-backend": "E2E_HOST='http://localhost:3000' yarn test-cypress-no-build --open",
-    "test-cypress-no-build": "yarn && CONFIG_FILE=frontend/test/__support__/e2e/cypress.json babel-node ./frontend/test/__runner__/run_cypress_tests.js",
+    "test-cypress-no-build": "yarn && babel-node ./frontend/test/__runner__/run_cypress_tests.js",
     "prepare": "husky install"
   },
   "lint-staged": {

yarn.lock

@@ -1518,6 +1518,11 @@ arch@^2.1.2:
   resolved "https://registry.yarnpkg.com/arch/-/arch-2.2.0.tgz#1bc47818f305764f23ab3306b0bfc086c5a29d11"
   integrity sha512-Of/R0wqp83cgHozfIYLbBMnej79U/SVGOOyuB3VVFv1NRM/PSFMK12x9KVtiYzJqmnU5WR2qp0Z5rHb7sWGnFQ==
+arg@^5.0.0:
+  version "5.0.0"
+  resolved "https://registry.yarnpkg.com/arg/-/arg-5.0.0.tgz#a20e2bb5710e82950a516b3f933fee5ed478be90"
+  integrity sha512-4P8Zm2H+BRS+c/xX1LrHw0qKpEhdlZjLCgWy+d78T9vqa2Z2SiD2wMrYuWIAFy5IZUD7nnNXroRttz+0RzlrzQ==
+
 argparse@^1.0.7:
   version "1.0.10"
   resolved "https://registry.yarnpkg.com/argparse/-/argparse-1.0.10.tgz#bcd6791ea5ae09725e17e5ad988134cd40b3d911"