diff --git a/frontend/test/__support__/cypress.js b/frontend/test/__support__/cypress.js
index efabda45ffbd8f7c31ff352870d75d4a0616cb1e..66994ba7b13f9345b57c15c47f856d916ce0c57a 100644
--- a/frontend/test/__support__/cypress.js
+++ b/frontend/test/__support__/cypress.js
@@ -81,6 +81,23 @@ export function openProductsTable() {
   cy.visit("/question/new?database=1&table=1");
 }
 
+export function setupLocalHostEmail() {
+  // Email info
+  cy.findByPlaceholderText("smtp.yourservice.com").type("localhost");
+  cy.findByPlaceholderText("587").type("1025");
+  cy.findByText("None").click();
+  // Leaves password and username blank
+  cy.findByPlaceholderText("metabase@yourcompany.com").type("test@local.host");
+
+  // *** Unnecessary click (Issue #12692)
+  cy.findByPlaceholderText("smtp.yourservice.com").click();
+
+  cy.findByText("Save changes").click();
+  cy.findByText("Changes saved!");
+
+  cy.findByText("Send test email").click();
+}
+
 // Find a text field by label text, type it in, then blur the field.
 // Commonly used in our Admin section as we auto-save settings.
 export function typeAndBlurUsingLabel(label, value) {
diff --git a/frontend/test/metabase-smoketest/admin_setup.cy.spec.js b/frontend/test/metabase-smoketest/admin_setup.cy.spec.js
index 74f1e419204a52e0cbeb57ab2edfd273c37c8016..97c1b7ebbf3ca7d0a4967d5f0683327a61459f30 100644
--- a/frontend/test/metabase-smoketest/admin_setup.cy.spec.js
+++ b/frontend/test/metabase-smoketest/admin_setup.cy.spec.js
@@ -5,6 +5,7 @@ import {
   signOut,
   signInAsNormalUser,
   signIn,
+  setupLocalHostEmail,
 } from "__support__/cypress";
 
 const new_user = {
@@ -67,22 +68,7 @@ describe("smoketest > admin_setup", () => {
       cy.findByText("Email address you want to use as the sender of Metabase.");
       cy.findByText("Sample Database").should("not.exist");
 
-      // Email info
-      cy.findByPlaceholderText("smtp.yourservice.com").type("localhost");
-      cy.findByPlaceholderText("587").type("1025");
-      cy.findByText("None").click();
-      // Leaves password and username blank
-      cy.findByPlaceholderText("metabase@yourcompany.com").type(
-        "test@local.host",
-      );
-
-      // *** Unnecessary click (Issue #12692)
-      cy.findByPlaceholderText("smtp.yourservice.com").click();
-
-      cy.findByText("Save changes").click();
-      cy.findByText("Changes saved!");
-
-      cy.findByText("Send test email").click();
+      setupLocalHostEmail();
 
       // *** Will fail if test works correctly:
       cy.wait(2000)
diff --git a/frontend/test/metabase/scenarios/README.md b/frontend/test/metabase/scenarios/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..574bc00394e0e896e31cff2b20e9120b893f3939
--- /dev/null
+++ b/frontend/test/metabase/scenarios/README.md
@@ -0,0 +1,5 @@
+# Metabase Scenarios
+
+## Running
+
+- If you are running tests that include `alert > email_alert`, run `python -m smtpd -n -c DebuggingServer localhost:1025`in terminal first for setting up email through your localhost
diff --git a/frontend/test/metabase/scenarios/alert/email_alert.cy.spec.js b/frontend/test/metabase/scenarios/alert/email_alert.cy.spec.js
new file mode 100644
index 0000000000000000000000000000000000000000..2d703e4d26e592e825a84d3ce753776a1a555155
--- /dev/null
+++ b/frontend/test/metabase/scenarios/alert/email_alert.cy.spec.js
@@ -0,0 +1,74 @@
+import {
+  restore,
+  signInAsAdmin,
+  setupLocalHostEmail,
+} from "../../../__support__/cypress";
+
+function setUpHourlyAlert(question_num) {
+  cy.visit(`/question/${question_num}`);
+  cy.get(".Icon-bell").click();
+  cy.findByText("Set up an alert").click();
+  cy.findByText("Daily").click();
+  cy.findByText("Hourly").click();
+}
+
+describe("scenarios > alert > email_alert", () => {
+  beforeEach(restore);
+  beforeEach(signInAsAdmin);
+
+  it("should have no alerts set up initially", () => {
+    cy.server();
+    cy.visit("/");
+
+    cy.request("/api/alert").then(response => {
+      expect(response.body).to.have.length(0);
+    });
+  });
+
+  describe.skip("alert set up", () => {
+    // NOTE: To run tests, first run `python -m smtpd -n -c DebuggingServer localhost:1025` in your terminal
+    beforeEach(() => {
+      cy.server();
+
+      cy.visit("/admin/settings/email");
+      cy.findByText("SMTP Host");
+      setupLocalHostEmail();
+    });
+
+    it("should work with email alerts toggled on", () => {
+      // Set up alert
+      setUpHourlyAlert(1);
+      cy.findByText("Done")
+        .click()
+        .then(() => {
+          cy.findByText("Sample Dataset");
+        });
+
+      // Check alert api is sending email
+      cy.request("/api/alert").then(response => {
+        expect(response.body[0].channels).to.have.length(1);
+        expect(response.body[0].channels[0].recipients).to.have.length(1);
+      });
+    });
+
+    it("should have email alerts toggled off (Issue #12349)", () => {
+      // Turn off email alerts during alert setup
+      setUpHourlyAlert(2);
+      cy.findByText("Email")
+        .parent()
+        .find("a")
+        .click();
+      cy.findByText("Done")
+        .click()
+        .then(() => {
+          cy.findAllByText("Orders");
+        });
+
+      // Check alert api is NOT sending email
+      cy.request("/api/alert").then(response => {
+        expect(response.body[0].channels).to.have.length(1);
+        expect(response.body[0].channels[0].recipients).to.equal("null");
+      });
+    });
+  });
+});