Enable enterprise cypress tests on release branch (#13809)

* Enable enterprise cypress tests (#13756)

* Get auditing.cy.spec to pass

* Skip custom drill through tests until we replace them

* A few updates to the whitelabel test case:
- Move custom colors into constant variable to reference throughout
- remove some 'it' blocks we didn't need to speed up tests
- Remove a few 'wait's
- Restructure tests to be a bit more in line with best practices

* Update the snippet permissions test:
- Create test for just creating snippet as admin
- Create test for inability to create snippet as user by default
- Quarantine the user snippet creation test

* Rename sandboxes

* Replace legacy ENABLE_ENTERPRISE_EDITION env variable with MB_EDITION

* Add MB_EDITION env variable to Cypress test run

* Fix admin settings `auth` cypress test (#13841) [ci skip]

* Fix `snippet` Cypress test for CI (#13843)

- related: https://github.com/metabase/metabase-enterprise/issues/543



Co-authored-by: Nemanja Glumac <31325167+nemanjaglumac@users.noreply.github.com>

.circleci/config.yml

@@ -692,6 +692,7 @@ jobs:
       <<: *Params
     executor: << parameters.e >>
     environment:
+      MB_EDITION: << parameters.edition >>
       CYPRESS_GROUP:  << parameters.cypress-group >>
       DISPLAY: ""
     steps:

enterprise/frontend/test/metabase-enterprise/audit/auditing.cy.spec.js

@@ -62,23 +62,20 @@ export function generateDashboards(users) {
 }
 
 describeWithToken("audit > auditing", () => {
-  before(restore);
   const users = ["admin", "normal"];
+  before(() => {
+    restore();
+    generateQuestions(users);
+    generateDashboards(users);
+  });
 
   describe("Generate data to audit", () => {
     beforeEach(signOut);
 
-    it("should create questions and dashboards", () => {
-      generateQuestions(users);
-      generateDashboards(users);
-    });
-
     it("should view a dashboard", () => {
       signIn("nodata");
       cy.visit("/collection/root?type=dashboard");
-      cy.wait(3000)
-        .findByText(users[1] + " test dash")
-        .click();
+      cy.findByText(users[1] + " test dash").click();
 
       cy.findByText("This dashboard is looking empty.");
       cy.findByText("My personal collection").should("not.exist");
@@ -87,18 +84,14 @@ describeWithToken("audit > auditing", () => {
     it("should view old question and new question", () => {
       signIn("nodata");
       cy.visit("/collection/root?type");
-      cy.wait(2000)
-        .findByText("Orders, Count")
-        .click();
+      cy.findByText("Orders, Count").click();
 
       cy.findByText("18,760");
 
       cy.visit("/collection/root?type");
-      cy.wait(2000)
-        .findByText(users[0] + " test q")
-        .click();
+      cy.findByText(users[0] + " test q").click();
 
-      cy.findByText("ID");
+      cy.get('[placeholder="ID"]');
     });
 
     it("should download a question", () => {
@@ -124,13 +117,20 @@ describeWithToken("audit > auditing", () => {
     it("should load the Overview tab", () => {
       cy.visit("/admin/audit/members/overview");
 
+      // We haven't created any new members yet so this should be empty
       cy.findByText("Active members and new members per day");
       cy.findByText("No results!");
-      cy.wait(1000)
-        .get(".LineAreaBarChart")
+
+      // Wait for both of the charts to show up
+      cy.get(".dc-chart").should("have.length", 2);
+
+      // For queries viewed, we have 2 users that haven't viewed anything
+      cy.get(".LineAreaBarChart")
         .first()
         .find("[width='0']")
         .should("have.length", 2);
+
+      // For queries created, we have 3 users that haven't created anything
       cy.get("svg")
         .last()
         .find("[width='0']")
@@ -183,7 +183,7 @@ describeWithToken("audit > auditing", () => {
       // Overview tab
       cy.visit("/admin/audit/schemas/overview");
       cy.get("svg").should("have.length", 2);
-      cy.wait(1000).findAllByText("Sample Dataset PUBLIC");
+      cy.findAllByText("Sample Dataset PUBLIC");
       cy.findAllByText("No results!").should("not.exist");
 
       // All schemas tab
@@ -197,7 +197,7 @@ describeWithToken("audit > auditing", () => {
       cy.visit("/admin/audit/tables/overview");
       cy.findByText("Most-queried tables");
       cy.findAllByText("No results!").should("not.exist");
-      cy.wait(1000).findAllByText("Sample Dataset PUBLIC ORDERS");
+      cy.findAllByText("Sample Dataset PUBLIC ORDERS");
 
       // *** Will fail when code below works again
       cy.findAllByText("Sample Dataset PUBLIC PRODUCTS").should("not.exist");
@@ -237,9 +237,7 @@ describeWithToken("audit > auditing", () => {
       // All questions tab
       cy.visit("/admin/audit/questions/all");
       cy.findByPlaceholderText("Question name");
-      cy.wait(1000)
-        .findAllByText("Sample Dataset")
-        .should("have.length", 5);
+      cy.findAllByText("Sample Dataset").should("have.length", 5);
       cy.findByText("normal test q");
       cy.findByText("Orders, Count, Grouped by Created At (year)");
       cy.findByText("4").should("not.exist");
@@ -275,9 +273,7 @@ describeWithToken("audit > auditing", () => {
 
       // All downloads tab
       cy.visit("/admin/audit/downloads/all");
-      cy.wait(2000)
-        .findByText("No results")
-        .should("not.exist");
+      cy.findByText("No results").should("not.exist");
       cy.get("tr")
         .last()
         .children()

enterprise/frontend/test/metabase-enterprise/custom_drill_through/drill_through.cy.spec.js

-import {
-  signIn,
-  signInAsAdmin,
-  restore,
-  modal,
-  describeWithToken,
-} from "__support__/cypress";
+import { signIn, signInAsAdmin, restore, modal } from "__support__/cypress";
 
-describeWithToken("drill through", () => {
+// Drill-through support has been replaced with custom dashboard destinations
+describe.skip("drill through", () => {
   before(restore);
 
   beforeEach(signInAsAdmin);

enterprise/frontend/test/metabase-enterprise/formatting/whitelabel.cy.spec.js

@@ -7,15 +7,20 @@ import {
   describeWithToken,
 } from "../../../../../frontend/test/__support__/cypress";
 
-const main_color = {
-  // brown - button, links, chart
-  hex: "8B572A",
-  rgb: "(139, 87, 42)",
+// Define colors that we use for whitelabeling
+// If rbg values exist, it's because we explicit test those
+const colors = {
+  primary: { hex: "8B572A", rgb: [139, 87, 42] },
+  nav: { hex: "284E07", rgb: [40, 78, 7] },
+  accent1: { hex: "417505" },
+  accent2: { hex: "7ED321" },
+  additional1: { hex: "B8E986" },
+  additional2: { hex: "50E3C2" },
+  additional3: { hex: "4A90E2" },
+  additional4: { hex: "082CBE" },
+  additional5: { hex: "F8E71C", rgb: [248, 231, 28] },
 };
 
-//green - nav bar
-const header_color = "rgb(40, 78, 7)";
-
 function changeThemeColor(location, colorhex) {
   cy.get("td")
     .eq(location)
@@ -40,40 +45,35 @@ function checkLogo() {
 describeWithToken("formatting > whitelabel", () => {
   before(restore);
 
-  describe("Changes to company name work", () => {
-    beforeEach(signOut);
-
-    it("should change company name", () => {
-      signInAsAdmin();
-      cy.visit("/admin/settings/whitelabel");
-      cy.findByPlaceholderText("Metabase")
-        .clear()
-        .type("Test Co");
-      // *** In html, is not text, only value
-      cy.findByText("Application Name").click();
-
-      cy.findByText("Saved");
-      cy.get("input").should("have.value", "Test Co");
-    });
-
-    it("should show new name on activity page as admin", () => {
-      signInAsAdmin();
-      cy.visit("/activity");
-      cy.findByText("Test Co is up and running.");
-      cy.findByText("Metabase is up and running.").should("not.exist");
-    });
-
-    it("should show new name when logged out", () => {
-      cy.visit("/");
-      cy.wait(2000).findByText("Sign in to Test Co");
-    });
-
-    it("should show new name on activity page as user", () => {
-      signInAsNormalUser();
-      cy.visit("/activity");
-      cy.findByText("Test Co is up and running.");
-      cy.findByText("Metabase is up and running.").should("not.exist");
-    });
+  it("should be able to change company name", () => {
+    signInAsAdmin();
+    cy.visit("/admin/settings/whitelabel");
+    cy.findByPlaceholderText("Metabase")
+      .clear()
+      .type("Test Co");
+    // *** In html, is not text, only value
+    cy.findByText("Application Name").click();
+
+    cy.findByText("Saved");
+    cy.get("input").should("have.value", "Test Co");
+    cy.log("Company name has been updated");
+
+    cy.log("New company show show up on activity page");
+    // signInAsAdmin();
+    cy.visit("/activity");
+    cy.findByText("Test Co is up and running.");
+    cy.findByText("Metabase is up and running.").should("not.exist");
+
+    cy.log("New company should show up when logged out");
+    signOut();
+    cy.visit("/");
+    cy.findByText("Sign in to Test Co");
+
+    cy.log("new company should show up as a normal user");
+    signInAsNormalUser();
+    cy.visit("/activity");
+    cy.findByText("Test Co is up and running.");
+    cy.findByText("Metabase is up and running.").should("not.exist");
   });
 
   describe("Changes to theme colors work", () => {
@@ -82,7 +82,7 @@ describeWithToken("formatting > whitelabel", () => {
       cy.visit("/admin/settings/whitelabel");
 
       // Select color with squares
-      changeThemeColor(1, main_color.hex);
+      changeThemeColor(1, colors.primary.hex);
 
       // Select color by entering rgb
       cy.get("td")
@@ -92,25 +92,25 @@ describeWithToken("formatting > whitelabel", () => {
         .find("input")
         .eq(1)
         .clear()
-        .type("40");
+        .type(colors.nav.rgb[0]);
       cy.get(".sketch-picker")
         .find("input")
         .eq(2)
         .clear()
-        .type("78");
+        .type(colors.nav.rgb[1]);
       cy.get(".sketch-picker")
         .find("input")
         .eq(3)
         .clear()
-        .type("7");
+        .type(colors.nav.rgb[2]);
       cy.findByText("Done").click();
 
       // Select colors with squares
-      changeThemeColor(9, "417505");
-      changeThemeColor(13, "7ED321");
-      changeThemeColor(17, "B8E986");
-      changeThemeColor(21, "50E3C2");
-      changeThemeColor(25, "4A90E2");
+      changeThemeColor(9, colors.accent1.hex);
+      changeThemeColor(13, colors.accent2.hex);
+      changeThemeColor(17, colors.additional1.hex);
+      changeThemeColor(21, colors.additional2.hex);
+      changeThemeColor(25, colors.additional3.hex);
 
       // Select color by typing hex code
       cy.get("td")
@@ -120,48 +120,74 @@ describeWithToken("formatting > whitelabel", () => {
         .find("input")
         .first()
         .clear()
-        .type("082CBE");
+        .type(colors.additional4.hex);
       cy.findByText("Done").click();
 
-      changeThemeColor(33, "F8E71C");
+      changeThemeColor(33, colors.additional5.hex);
 
       cy.get(".Icon-close").should("have.length", 10);
     });
 
-    it("should show color changes on admin's dashboard", () => {
-      signInAsAdmin();
-      cy.visit("/");
-      cy.get(`[style='background-color: ${header_color};']`);
-    });
-
-    it("should show color changes when signed out", () => {
+    it("should show color changes", () => {
       signOut();
       cy.visit("/");
-      cy.get(
-        `[style='width: 16px; height: 16px; background-color: rgb${main_color.rgb}; border: 2px solid rgb${main_color.rgb};']`,
+      cy.contains("Sign in");
+
+      // Note that if we have modified the logo, the entire background turns the brand color.
+      // But if we _haven't_, as is the case now, then the existing logo is branded
+      // As is the "Remember me" and "Sign in" inputs
+      cy.get(".Icon").should(
+        "have.css",
+        "color",
+        `rgb(${colors.primary.rgb.join(", ")})`,
       );
-    });
 
-    it("should show color changes on user's dashboard", () => {
+      cy.findByLabelText("Email address").type("some@email.com");
+      cy.findByLabelText("Password").type("1234");
+      cy.get(".Button--primary").should(
+        "have.css",
+        "background-color",
+        `rgb(${colors.primary.rgb.join(", ")})`,
+      );
+
+      cy.log("Normal users should have a green header");
       signInAsNormalUser();
       cy.visit("/");
-      cy.get(`[style='background-color: ${header_color};']`);
+      cy.get(".Nav").should(
+        "have.css",
+        "background-color",
+        `rgb(${colors.nav.rgb.join(", ")})`,
+      );
+
+      cy.log(
+        "Admin users should also have a green header, but yellow in the admin panel",
+      );
+      signInAsAdmin();
+      cy.visit("/");
+      cy.get(".Nav").should(
+        "have.css",
+        "background-color",
+        `rgb(${colors.nav.rgb.join(", ")})`,
+      );
+      cy.visit("/admin");
+      cy.get(".Nav").should(
+        "have.css",
+        "background-color",
+        `rgb(${colors.additional5.rgb.join(", ")})`,
+      );
     });
 
     it.skip("should show color changes reflected in q visualizations (metabase-enterprise #470)", () => {
       // *** Test should pass when issue #470 is resolved
       signInAsNormalUser();
       openOrdersTable();
-      cy.wait(3000)
-        .findAllByText("Summarize")
+      cy.findAllByText("Summarize")
         .first()
         .click();
-      cy.wait(1000)
-        .findByText("Price")
-        .click();
+      cy.findByText("Price").click();
       cy.findByText("Done").click();
 
-      cy.get(`div[fill='#${main_color.hex};']`);
+      cy.get(`div[fill='#${colors.primary.hex};']`);
       cy.get(`rect[fill='#509EE3']`).should("not.exist");
     });
   });
@@ -210,36 +236,26 @@ describeWithToken("formatting > whitelabel", () => {
     });
   });
 
-  describe("Changes to favicon work", () => {
-    it("should add a favicon", () => {
-      signInAsAdmin();
-      cy.visit("/admin/settings/whitelabel");
-
-      cy.server();
-
-      cy.findByPlaceholderText("frontend_client/favicon.ico").type(
-        "https://cdn.ecosia.org/assets/images/ico/favicon.ico",
-      );
-      cy.get("ul")
-        .eq(2)
-        .click("right");
-      cy.wait(10).findByText("Saved");
-
-      checkFavicon();
-    });
-
-    it("should reflect favicon change in API", () => {
-      signInAsAdmin();
-      cy.visit("/");
-      checkFavicon();
-    });
-
-    it("should reflect favicon change in HTML", () => {
-      signInAsNormalUser();
-      cy.visit("/");
-      cy.get('head link[rel="icon"]')
-        .get('[href="https://cdn.ecosia.org/assets/images/ico/favicon.ico"]')
-        .should("have.length", 1);
-    });
+  it("should add a custom favicon", () => {
+    signInAsAdmin();
+    cy.visit("/admin/settings/whitelabel");
+
+    cy.server();
+
+    cy.findByPlaceholderText("frontend_client/favicon.ico").type(
+      "https://cdn.ecosia.org/assets/images/ico/favicon.ico",
+    );
+    cy.get("ul")
+      .eq(2)
+      .click("right");
+    cy.findByText("Saved");
+    checkFavicon();
+
+    cy.log("New favicon should show up in user's HTML");
+    signInAsNormalUser();
+    cy.visit("/");
+    cy.get('head link[rel="icon"]')
+      .get('[href="https://cdn.ecosia.org/assets/images/ico/favicon.ico"]')
+      .should("have.length", 1);
   });
 });

enterprise/frontend/test/metabase-enterprise/snippets/snippet-permissions.cy.spec.js

@@ -10,7 +10,105 @@ import {
 
 describeWithToken("scenarios > question > snippets", () => {
   before(restore);
-  beforeEach(signInAsNormalUser);
+  beforeEach(signInAsAdmin);
+
+  it("can create a snippet", () => {
+    cy.visit("/question/new");
+    cy.contains("Native query").click();
+    cy.get(".Icon-snippet").click();
+    cy.contains("Create a snippet").click();
+    modal().within(() => {
+      cy.findByLabelText("Enter some SQL here so you can reuse it later").type(
+        "SELECT 'a snippet darkly'",
+      );
+      cy.findByLabelText("Give your snippet a name").type("night snippet");
+      cy.contains("Save").click();
+    });
+    cy.get(".Icon-play")
+      .first()
+      .click();
+    cy.get(".ScalarValue").contains("a snippet darkly");
+  });
+
+  it("can not create a snippet as a user by default", () => {
+    // Note that this is expected behavior, but a little weird because
+    // users have to be granted explicit access.
+    // See metabase-enterprise#543 for more details
+
+    signInAsNormalUser();
+
+    cy.request({
+      method: "POST",
+      url: "/api/native-query-snippet",
+      body: {
+        content: "SELECT 'a snippet in light'",
+        name: "light snippet",
+        collection_id: null,
+      },
+      failOnStatusCode: false,
+    }).then(resp => {
+      expect(resp.status).to.equal(403);
+    });
+  });
+
+  // [quarantine] because the popover click action is very flaky.
+  it.skip("can create a snippet once the admin has granted access", () => {
+    // See metabase-enterprise#543 for more details
+    // This is kind of a UX issue where the admin has to:
+    // - First create a snippet
+    // - Then grant All Users access to snippets
+
+    // create snippet via API
+    cy.request("POST", "/api/native-query-snippet", {
+      content: "SELECT 'a snippet darkly'",
+      name: "543 - admin snippet",
+      collection_id: null,
+    });
+
+    // Grant access
+    cy.visit("/question/new");
+    cy.contains("Native query").click();
+    cy.get(".Icon-snippet").click();
+
+    sidebar()
+      .find(".Icon-ellipsis")
+      .click({ force: true });
+    popover().within(() => cy.findByText("Change permissions").click());
+    modal().within(() => {
+      cy.findByText("Permissions for Top folder");
+      cy.contains("All Users");
+      cy.get(".ReactVirtualized__Grid .Icon-close")
+        .first()
+        .click();
+    });
+    // The click action is very flaky, sometimes it doesn't click the right thing
+    popover()
+      .contains("Grant Edit access")
+      .click();
+    modal()
+      .contains("Save")
+      .click();
+    // Now the user should be able to create a snippet
+    signInAsNormalUser();
+
+    cy.request({
+      method: "POST",
+      url: "/api/native-query-snippet",
+      body: {
+        content: "SELECT 'a snippet in light'",
+        name: "543 - user snippet",
+        collection_id: null,
+      },
+      failOnStatusCode: false,
+    }).then(resp => {
+      expect(resp.status).to.equal(200);
+    });
+
+    cy.reload();
+    cy.get(".Icon-snippet").click();
+    cy.contains("543 - admin snippet");
+    cy.contains("543 - user snippet");
+  });
 
   it("should let you create a snippet folder and move a snippet into it", () => {
     cy.visit("/question/new");
@@ -59,23 +157,4 @@ describeWithToken("scenarios > question > snippets", () => {
     cy.findByText("my favorite snippets").click();
     cy.findByText("snippet 1");
   });
-
-  it("should allow updating snippet folder permissions", () => {
-    signInAsAdmin();
-    cy.visit("/question/new");
-    cy.contains("Native query").click();
-    cy.get(".Icon-snippet").click();
-
-    sidebar()
-      .findByText("my favorite snippets")
-      .parent()
-      .parent()
-      .find(".Icon-ellipsis")
-      .click({ force: true });
-    popover().within(() => cy.findByText("Change permissions").click());
-    modal().within(() => {
-      cy.findByText("Permissions for this folder");
-    });
-    // TODO: incomplete
-  });
 });

frontend/test/__runner__/backend.js

@@ -65,7 +65,7 @@ export const BackendResource = createSharedResource("BackendResource", {
             MB_JETTY_PORT: server.port,
             MB_ENABLE_TEST_ENDPOINTS: "true",
             MB_PREMIUM_EMBEDDING_TOKEN:
-              (process.env["ENABLE_ENTERPRISE_EDITION"] === "true" &&
+              (process.env["MB_EDITION"] === "ee" &&
                 process.env["ENTERPRISE_TOKEN"]) ||
               undefined,
           },

frontend/test/__runner__/run_cypress_tests.js

@@ -77,7 +77,7 @@ const init = async () => {
   // These env vars provide the token to the backend.
   // If they're not present, we skip some tests that depend on a valid token.
   const hasEnterpriseToken =
-    process.env["ENTERPRISE_TOKEN"] && process.env["ENABLE_ENTERPRISE_EDITION"];
+    process.env["ENTERPRISE_TOKEN"] && process.env["MB_EDITION"] === "ee";
 
   const cypressProcess = spawn(
     "yarn",

frontend/test/metabase/scenarios/admin/settings/settings.cy.spec.js

@@ -46,10 +46,9 @@ describe("scenarios > admin > settings", () => {
     // Ported from `SettingsAuthenticationOptions.e2e.spec.js`
     // Google sign in
     cy.visit("/admin/settings/authentication");
-    cy.findByText("Sign in with Google");
-    cy.findAllByText("Configure")
-      .first()
-      .click();
+
+    configureAuth("Sign in with Google");
+
     cy.contains(
       "To allow users to sign in with Google you'll need to give Metabase a Google Developers console application client ID.",
     );
@@ -58,10 +57,9 @@ describe("scenarios > admin > settings", () => {
 
     // SSO
     cy.visit("/admin/settings/authentication");
-    cy.findByText("LDAP").click();
-    cy.findAllByText("Configure")
-      .last()
-      .click();
+
+    configureAuth("LDAP");
+
     cy.findByText("LDAP Authentication");
     cy.findByText("User Schema");
     cy.findByText("Save changes");
@@ -310,3 +308,10 @@ describe("scenarios > admin > settings", () => {
     });
   });
 });
+
+function configureAuth(providerTitle) {
+  cy.findByText(providerTitle)
+    .closest(".rounded.bordered")
+    .contains("Configure")
+    .click();
+}

frontend/test/metabase/scenarios/question/snippets.cy.spec.js

-import { signInAsNormalUser, restore, modal } from "__support__/cypress";
+import { signInAsAdmin, restore, modal } from "__support__/cypress";
 
 // HACK which lets us type (even very long words) without losing focus
 // this is needed for fields where autocomplete suggestions are enabled
@@ -12,9 +12,14 @@ function _clearAndIterativelyTypeUsingLabel(label, string) {
   }
 }
 
+// NOTE: - Had to change user role to "admin" on 2020-11-19.
+//       - Normal users don't have permission to create/edit snippets in `ee` version.
+//       - CI runs this test twice (both contexts), so it fails on `ee`.
+//       - There is a related issue: https://github.com/metabase/metabase-enterprise/issues/543
+// TODO: Once the above issue is (re)solved, change back to `signInAsNormalUser`
 describe("scenarios > question > snippets", () => {
   before(restore);
-  beforeEach(signInAsNormalUser);
+  beforeEach(signInAsAdmin);
 
   it("should let you create and use a snippet", () => {
     cy.visit("/question/new");