E2E test snapshot/restore (#11564)

E2E (for Cypress) snapshot/restore

frontend/src/metabase/components/EntityMenuItem.jsx

@@ -51,6 +51,7 @@ const LinkMenuItem = ({ children, link, onClose, event, externalLink }) => (
     target={externalLink ? "_blank" : null}
     onClick={onClose}
     data-metabase-event={event}
+    style={{ display: "block" }}
   >
     {children}
   </Link>

frontend/test/__runner__/backend.js

@@ -63,6 +63,7 @@ export const BackendResource = createSharedResource("BackendResource", {
             MB_DB_FILE: server.dbFile,
             MB_JETTY_HOST: "0.0.0.0",
             MB_JETTY_PORT: server.port,
+            MB_ENABLE_TEST_ENDPOINTS: "true",
           },
           stdio:
             process.env["DISABLE_LOGGING"] ||

frontend/test/__runner__/run_cypress_tests.js

@@ -5,14 +5,7 @@ import chalk from "chalk";
 // Use require for BackendResource to run it after the mock afterAll has been set
 const BackendResource = require("./backend.js").BackendResource;
 
-// Backend that uses a test fixture database
-const serverWithTestDbFixture = BackendResource.get({
-  dbKey: "/cypress_db_fixture.db",
-});
-const testFixtureBackendHost = serverWithTestDbFixture.host;
-
-const serverWithPlainDb = BackendResource.get({ dbKey: "" });
-const plainBackendHost = serverWithPlainDb.host;
+const server = BackendResource.get({ dbKey: "" });
 
 const userArgs = process.argv.slice(2);
 const isOpenMode = userArgs[0] === "--open";
@@ -28,34 +21,6 @@ function readFile(fileName) {
   });
 }
 
-const login = async (apiHost, user) => {
-  const loginFetchOptions = {
-    method: "POST",
-    headers: new Headers({
-      Accept: "application/json",
-      "Content-Type": "application/json",
-    }),
-    body: JSON.stringify(user),
-  };
-  const result = await fetch(apiHost + "/api/session", loginFetchOptions);
-
-  let resultBody = null;
-  try {
-    resultBody = await result.text();
-    resultBody = JSON.parse(resultBody);
-  } catch (e) {}
-
-  if (result.status >= 200 && result.status <= 299) {
-    console.log(`Successfully created a shared login with id ${resultBody.id}`);
-    return resultBody;
-  } else {
-    const error = { status: result.status, data: resultBody };
-    console.log("A shared login attempt failed with the following error:");
-    console.log(error, { depth: null });
-    throw error;
-  }
-};
-
 const init = async () => {
   if (!isOpenMode) {
     console.log(
@@ -85,50 +50,19 @@ const init = async () => {
     process.exit(1);
   }
 
-  console.log(
-    chalk.bold("1/4 Starting first backend with test H2 database fixture"),
-  );
-  console.log(
-    chalk.cyan(
-      "You can update the fixture by running a local instance against it:\n`MB_DB_TYPE=h2 MB_DB_FILE=frontend/test/__runner__/cypress_db_fixture.db lein run`",
-    ),
-  );
-  await BackendResource.start(serverWithTestDbFixture);
-
-  console.log(chalk.bold("2/4 Starting second backend with plain database"));
-  await BackendResource.start(serverWithPlainDb);
-
-  console.log(chalk.bold("3/4 Creating a shared login session for backend 1"));
-  const sharedAdminLoginSession = await login(testFixtureBackendHost, {
-    username: "bob@metabase.com",
-    password: "12341234",
-  });
-  const sharedNormalLoginSession = await login(testFixtureBackendHost, {
-    username: "robert@metabase.com",
-    password: "12341234",
-  });
-
-  console.log(chalk.bold("4/4 Starting Cypress"));
-
-  const serializedEnv = Object.entries({
-    TEST_FIXTURE_SHARED_ADMIN_LOGIN_SESSION_ID: sharedAdminLoginSession.id,
-    TEST_FIXTURE_SHARED_NORMAL_LOGIN_SESSION_ID: sharedNormalLoginSession.id,
-    PLAIN_DB_HOST: plainBackendHost,
-  })
-    .map(a => a.join("="))
-    .join(",");
+  console.log(chalk.bold("Starting backend"));
+  await BackendResource.start(server);
 
+  console.log(chalk.bold("Starting Cypress"));
   const cypressProcess = spawn(
     "yarn",
     [
       "cypress",
       isOpenMode ? "open" : "run",
       "--config-file",
-      "frontend/test/cypress.json",
+      process.env["CONFIG_FILE"],
       "--config",
-      `baseUrl=${testFixtureBackendHost}`,
-      "--env",
-      serializedEnv,
+      `baseUrl=${server.host}`,
       ...(process.env["CI"]
         ? [
             "--reporter",
@@ -148,8 +82,7 @@ const init = async () => {
 
 const cleanup = async (exitCode = 0) => {
   console.log(chalk.bold("Cleaning up..."));
-  await BackendResource.stop(serverWithTestDbFixture);
-  await BackendResource.stop(serverWithPlainDb);
+  await BackendResource.stop(server);
   process.exit(exitCode);
 };
 
@@ -163,10 +96,5 @@ const launch = () =>
 
 launch();
 
-process.on("SIGTERM", () => {
-  cleanup();
-});
-
-process.on("SIGINT", () => {
-  cleanup();
-});
+process.on("SIGTERM", cleanup);
+process.on("SIGINT", cleanup);

frontend/test/__support__/cypress.js

+export const ADMIN_CREDS = {
+  username: "bob@metabase.com",
+  password: "12341234",
+};
+
+export const NORMAL_USER_CREDS = {
+  username: "robert@metabase.com",
+  password: "12341234",
+};
+
 export function signInAsAdmin() {
-  const sessionId = Cypress.env("TEST_FIXTURE_SHARED_ADMIN_LOGIN_SESSION_ID");
-  cy.setCookie("metabase.SESSION", sessionId);
+  cy.request("POST", "/api/session", ADMIN_CREDS);
 }
 
 export function signInAsNormalUser() {
-  const sessionId = Cypress.env("TEST_FIXTURE_SHARED_NORMAL_LOGIN_SESSION_ID");
-  cy.setCookie("metabase.SESSION", sessionId);
+  cy.request("POST", "/api/session", NORMAL_USER_CREDS);
 }
 
-export const plainDbHost = Cypress.env("PLAIN_DB_HOST");
+export function snapshot(name) {
+  cy.request("POST", `/api/testing/snapshot/${name}`);
+}
+export function restore(name = "default") {
+  cy.request("POST", `/api/testing/restore/${name}`);
+}
 
 Cypress.on("uncaught:exception", (err, runnable) => false);

frontend/test/cypress-snapshots.json

+{
+	"testFiles": "**/*.cy.snap.js",
+	"pluginsFile": "frontend/test/cypress-plugins.js",
+	"integrationFolder": "frontend/test",
+	"supportFile": "frontend/test/__support__/cypress.js"
+}

frontend/test/metabase/admin/people/people.cy.spec.js

-import { signInAsAdmin } from "__support__/cypress";
+import { signInAsAdmin, restore } from "__support__/cypress";
 
 describe("admin/people", () => {
+  before(restore);
   beforeEach(signInAsAdmin);
 
   describe("user management", () => {

frontend/test/metabase/admin/settings/settings.cy.spec.js

-import { signInAsAdmin } from "__support__/cypress";
+import { signInAsAdmin, restore } from "__support__/cypress";
 
 describe("admin/settings", () => {
+  before(restore);
   beforeEach(signInAsAdmin);
 
   it("should save a setting", () => {

frontend/test/metabase/dashboard/dashboard.cy.spec.js

-import { signInAsAdmin } from "__support__/cypress";
+import { signInAsAdmin, restore } from "__support__/cypress";
 
 describe("dashboard", () => {
+  before(restore);
   beforeEach(signInAsAdmin);
 
   it("should have the correct embed snippet", () => {
@@ -9,13 +10,13 @@ describe("dashboard", () => {
     cy.contains(/Embed this .* in an application/).click();
     cy.contains("Code").click();
 
-    const JS_CODE = `// you will need to install via 'npm install jsonwebtoken' or in your package.json
+    const JS_CODE = new RegExp(
+      `// you will need to install via 'npm install jsonwebtoken' or in your package.json
 
 var jwt = require("jsonwebtoken");
 
-var METABASE_SITE_URL = "http://localhost:3000";
-var METABASE_SECRET_KEY = "e893e786425e7604263d8d9590937e7a59d41d940fe99d529690b0e2cd3662a5";
-
+var METABASE_SITE_URL = "http://localhost:PORTPORTPORT";
+var METABASE_SECRET_KEY = "KEYKEYKEY";
 var payload = {
   resource: { dashboard: 1 },
   params: {},
@@ -24,8 +25,12 @@ var payload = {
 var token = jwt.sign(payload, METABASE_SECRET_KEY);
 
 var iframeUrl = METABASE_SITE_URL + "/embed/dashboard/" + token + "#bordered=true&titled=true";`
-      .split("\n")
-      .join("");
+        .split("\n")
+        .join("")
+        .replace(/[-\/\\^$*+?.()|[\]{}]/g, "\\$&")
+        .replace("KEYKEYKEY", ".*")
+        .replace("PORTPORTPORT", ".*"),
+    );
 
     const IFRAME_CODE = `<iframe
     src="{{iframeUrl}}"
@@ -39,7 +44,8 @@ var iframeUrl = METABASE_SITE_URL + "/embed/dashboard/" + token + "#bordered=tru
 
     cy.get(".ace_content")
       .first()
-      .should("have.text", JS_CODE);
+      .invoke("text")
+      .should("match", JS_CODE);
     cy.get(".ace_content")
       .last()
       .should("have.text", IFRAME_CODE);

frontend/test/metabase/home/Overworld.cy.spec.js

-import { signInAsAdmin, signInAsNormalUser } from "__support__/cypress";
+import {
+  signInAsAdmin,
+  signInAsNormalUser,
+  restore,
+} from "__support__/cypress";
 
 describe("homepage", () => {
+  before(restore);
+
   describe("content management", () => {
     describe("as admin", () => {
       beforeEach(() => {

frontend/test/metabase/public/signin.cy.spec.js

+import { restore } from "__support__/cypress";
 describe("sign in", () => {
+  before(restore);
+
   it("should display an error for incorrect passwords", () => {
     cy.visit("/");
 

frontend/test/metabase/query_builder/simple-question.cy.spec.js

-import { signInAsAdmin } from "__support__/cypress";
+import { signInAsAdmin, restore } from "__support__/cypress";
 
 describe("new question", () => {
+  before(restore);
   beforeEach(signInAsAdmin);
 
   it("should count all orders", () => {

frontend/test/metabase/reference/databases.cy.spec.js

-import { signInAsAdmin } from "__support__/cypress";
+import { signInAsAdmin, restore } from "__support__/cypress";
 
 describe("sample database reference", () => {
+  before(restore);
   beforeEach(signInAsAdmin);
 
   it("should see the listing", () => {
@@ -48,12 +49,5 @@ describe("sample database reference", () => {
       .type("My definitely profitable business");
     cy.contains("Save").click();
     cy.contains("My definitely profitable business");
-
-    // reset
-    cy.contains("Edit").click();
-    cy.get(".wrapper input")
-      .clear()
-      .type("Sample Dataset");
-    cy.contains("Save").click();
   });
 });

frontend/test/metabase/reference/guide.cy.spec.js

-import { signInAsAdmin } from "__support__/cypress";
+import { signInAsAdmin, restore } from "__support__/cypress";
 describe("getting started guide", () => {
+  before(restore);
   beforeEach(signInAsAdmin);
   it("should render", () => {
     cy.visit("reference");

frontend/test/metabase/scenarios/query_builder.cy.spec.js

-import { signInAsAdmin } from "__support__/cypress";
+import { signInAsAdmin, restore } from "__support__/cypress";
 
 describe("query builder", () => {
+  before(restore);
   beforeEach(signInAsAdmin);
 
   describe("browse data", () => {

frontend/test/metabase/setup/signup.cy.spec.js

 import path from "path";
-import { plainDbHost } from "__support__/cypress";
+import { restore } from "__support__/cypress";
 
 describe("setup wizard", () => {
-  before(() => {
-    Cypress.config("baseUrl", plainDbHost);
-  });
+  before(() => restore("blank"));
 
   it("should allow you to sign up", () => {
     // intial redirection and welcome page

frontend/test/snapshot-creators/default.cy.snap.js

+import {
+  snapshot,
+  restore,
+  ADMIN_CREDS,
+  NORMAL_USER_CREDS,
+} from "__support__/cypress";
+
+describe("default", () => {
+  it("default", () => {
+    snapshot("blank");
+    setup();
+    updateSettings();
+    createQuestionAndDashboard();
+    addUser();
+    snapshot("default");
+    restore("blank");
+  });
+});
+
+function setup() {
+  cy.visit("/");
+  cy.contains("Let's get started").click();
+
+  // User
+  const { username, password } = ADMIN_CREDS;
+  cy.get('input[name="first_name"]').type("Bobby");
+  cy.get('input[name="last_name"]').type("Tables");
+  cy.get('input[name="email"]').type(username);
+  cy.get('input[name="password"]').type(password);
+  cy.get('input[name="password_confirm"]').type(password);
+  cy.get('input[name="site_name"]').type("Epic Team");
+  cy.contains("Next").click();
+
+  // Database
+  cy.contains("I'll add my data later").click();
+
+  // Data Preferences
+  cy.contains("Allow Metabase to anonymously collect usage events")
+    .parents(".Form-field")
+    .find("a")
+    .click();
+  cy.contains("Next").click();
+
+  cy.contains("Take me to Metabase").click();
+}
+
+function updateSettings() {
+  cy.visit("/admin/settings/public_sharing");
+  cy.contains("Disabled")
+    .prev()
+    .click();
+  cy.contains("Saved");
+
+  cy.visit("/admin/settings/embedding_in_other_applications");
+  cy.contains(/Enable/).click();
+  cy.contains("Saved");
+
+  // update the Sample db connection string so it is valid in both CI and locally
+  cy.visit("/admin/databases/1");
+  cy.contains("Connection String")
+    .next()
+    .type("{SelectAll}./resources/sample-dataset.db;USER=GUEST;PASSWORD=guest");
+  cy.contains("Save").click();
+  cy.contains("Successfully saved!");
+}
+
+function addUser() {
+  cy.visit("/admin/people");
+  cy.contains("Add someone").click();
+
+  const typeFieldInModal = (label, text) =>
+    cy
+      .get(".ModalContent")
+      .contains(label)
+      .next()
+      .type(text);
+
+  typeFieldInModal("First name", "Robert");
+  typeFieldInModal("Last name", "Tableton");
+  const { username, password } = NORMAL_USER_CREDS;
+  typeFieldInModal("Email", username);
+  cy.contains("Create").click();
+  cy.contains("Show").click();
+  cy.contains("Temporary Password")
+    .parent()
+    .next()
+    .invoke("val")
+    .as("tempPassword");
+
+  // Log out of the admin account, so we can change the password
+  cy.contains("Done").click();
+  cy.get(".Icon-gear")
+    .last()
+    .click();
+  cy.contains("Sign out").click();
+
+  // On logout, the signin page briefly flashes before the app reloads and
+  // displays it permanently. We need to wait for that reload so we don't start
+  // typing too soon.
+  cy.wait(1000);
+
+  // log into the normal user account using the temp password
+  cy.contains("Email address")
+    .next()
+    .type(username);
+  cy.get("@tempPassword").then(t =>
+    cy
+      .contains("Password")
+      .next()
+      .type(t),
+  );
+  cy.get(".Button").click();
+
+  // go to update password form
+  cy.get(".Icon-gear").click();
+  cy.contains("Account settings").click();
+  cy.contains("Password").click();
+
+  // update password
+  cy.get("@tempPassword").then(t =>
+    cy.get(`input[name="old_password"]`).type(t),
+  );
+  cy.get(`input[name="password"]`)
+    .first()
+    .type(password);
+  cy.get(`input[name="password"]`)
+    .last()
+    .type(password);
+  cy.contains("Save").click();
+  cy.contains("Password updated successfully!");
+}
+
+function createQuestionAndDashboard() {
+  cy.visit("/question/new");
+  cy.contains("Simple question").click();
+  cy.contains("Orders").click();
+  cy.contains("37.65");
+  cy.contains("Summarize").click();
+  cy.contains("Done").click();
+  cy.contains("18,760");
+  cy.contains("Save").click(); // open save modal
+  cy.get(".ModalContent")
+    .contains(/^Save$/)
+    .click(); // save question
+  cy.contains("Saved!");
+  cy.contains("Yes please!").click();
+  cy.contains("Create a new dashboard").click();
+  cy.contains("Name")
+    .next()
+    .type("orders in a dashboard");
+  cy.get(".ModalContent")
+    .contains("Create")
+    .click();
+  cy.contains("You are editing a dashboard");
+  cy.contains("Save").click();
+  cy.get(".EditHeader").should("have.length", 0);
+}

package.json

@@ -212,7 +212,8 @@
     "ci-backend": "lein docstring-checker && lein bikeshed && lein eastwood && lein test",
     "test-cypress-headless": "yarn build && ./bin/build-for-test && yarn test-cypress-no-build",
     "test-cypress-open": "./bin/build-for-test && yarn test-cypress-no-build --open",
-    "test-cypress-no-build": "yarn && babel-node ./frontend/test/__runner__/run_cypress_tests.js"
+    "test-cypress-no-build": "yarn && CONFIG_FILE=frontend/test/cypress.json babel-node ./frontend/test/__runner__/run_cypress_tests.js",
+    "create-cypress-snapshots": "yarn && CONFIG_FILE=frontend/test/cypress-snapshots.json babel-node ./frontend/test/__runner__/run_cypress_tests.js"
   },
   "lint-staged": {
     "frontend/**/*.{js,jsx,css}": [

src/metabase/api/routes.clj

@@ -31,10 +31,12 @@
              [slack :as slack]
              [table :as table]
              [task :as task]
+             [testing :as testing]
              [tiles :as tiles]
              [transform :as transform]
              [user :as user]
              [util :as util]]
+            [metabase.config :as config]
             [metabase.middleware
              [auth :as middleware.auth]
              [exceptions :as middleware.exceptions]]
@@ -87,6 +89,9 @@
   (context "/slack"                [] (+auth slack/routes))
   (context "/table"                [] (+auth table/routes))
   (context "/task"                 [] (+auth task/routes))
+  (context "/testing"              [] (if (config/config-bool :mb-enable-test-endpoints)
+                                        testing/routes
+                                        (fn [_ respond _] (respond nil))))
   (context "/tiles"                [] (+auth tiles/routes))
   (context "/transform"            [] (+auth transform/routes))
   (context "/user"                 [] (+auth user/routes))