diff --git a/src/metabase/models/setting.clj b/src/metabase/models/setting.clj
index 4b06b18adf6e44d66ef4fa887a00ff52c30cdde2..47ba4e6605a035830916b99d7c678d451f7675e8 100644
--- a/src/metabase/models/setting.clj
+++ b/src/metabase/models/setting.clj
@@ -878,14 +878,19 @@
   [description-form]
   (when-not (valid-trs-or-tru? description-form)
     ;; this doesn't need to be i18n'ed because it's a compile-time error.
-    (throw (ex-info (str "defsetting docstrings must be an *deferred* i18n form unless the Setting has"
-                         " `:visibilty` `:internal` or `:setter` `:none`."
+    (throw (ex-info (str "defsetting docstrings must be a *deferred* i18n form unless the Setting has"
+                         " `:visibilty` `:internal`, `:setter` `:none`, or is defined in a test namespace."
                          (format " Got: ^%s %s"
                                  (some-> description-form class (.getCanonicalName))
                                  (pr-str description-form)))
                     {:description-form description-form})))
   description-form)
 
+(defn- in-test?
+  "Is `defsetting` currently being used in a test namespace?"
+  []
+  (str/ends-with? (ns-name *ns*) "-test"))
+
 (defmacro defsetting
   "Defines a new Setting that will be added to the DB at some point in the future.
   Conveniently can be used as a getter/setter as well
@@ -977,7 +982,8 @@
          ;; and `exciting!!` for the setter.
          (not (str/includes? (name setting-symbol) "!"))]}
   (let [description               (if (or (= (:visibility options) :internal)
-                                          (= (:setter options) :none))
+                                          (= (:setter options) :none)
+                                          (in-test?))
                                     description
                                     (validate-description-form description))
         definition-form           (assoc options
diff --git a/test/metabase/models/setting_test.clj b/test/metabase/models/setting_test.clj
index 222f22052fd87ef13ca532e24d671e77f7dac3a2..3effaed6487519f400f7d4dd85a54cbbcf13a8ff 100644
--- a/test/metabase/models/setting_test.clj
+++ b/test/metabase/models/setting_test.clj
@@ -1,6 +1,7 @@
 (ns metabase.models.setting-test
   (:require
    [clojure.test :refer :all]
+   [clojure.walk :as walk]
    [environ.core :as env]
    [medley.core :as m]
    [metabase.db.query :as mdb.query]
@@ -21,14 +22,14 @@
 ;; ## TEST SETTINGS DEFINITIONS
 
 (defsetting test-setting-1
-  (deferred-tru "Test setting - this only shows up in dev (1)"))
+  "Test setting - this only shows up in dev (1)")
 
 (defsetting test-setting-2
-  (deferred-tru "Test setting - this only shows up in dev (2)")
+  "Test setting - this only shows up in dev (2)"
   :default "[Default Value]")
 
 (defsetting test-setting-3
-  (deferred-tru "Test setting - this only shows up in dev (3)")
+  "Test setting - this only shows up in dev (3)"
   :visibility :internal)
 
 (defsetting test-boolean-setting
@@ -37,7 +38,7 @@
   :type :boolean)
 
 (defsetting test-json-setting
-  (deferred-tru "Test setting - this only shows up in dev (4)")
+  "Test setting - this only shows up in dev (4)"
   :type :json)
 
 (defsetting test-csv-setting
@@ -963,3 +964,14 @@
                  Exception
                  #"Wrong :default type: got \^clojure\.lang\.Keyword :green-friend, but expected a java\.lang\.String"
                  (validate tag value)))))))))
+
+(deftest validate-description-translation-test
+  (with-redefs [metabase.models.setting/in-test? (constantly false)]
+    (testing "When not in a test, defsetting descriptions must be i18n'ed"
+      (try
+        (walk/macroexpand-all
+         `(defsetting ~'test-asdf-asdf-asdf
+            "untranslated description"))
+        (catch Exception e
+          (is (re-matches #"defsetting docstrings must be a \*deferred\* i18n form.*"
+                          (:cause (Throwable->map e)))))))))