Add test profiling info & don't suppress dataset load output [ci drivers]

test/metabase/driver/generic_sql_test.clj

@@ -8,6 +8,7 @@
             [metabase.test.data :as data :refer :all]
             [metabase.test.data.datasets :as datasets]
             [metabase.test.util.log :as tu.log]
+            [metabase.util.date :as du]
             [toucan.db :as db])
   (:import metabase.driver.h2.H2Driver))
 
@@ -16,12 +17,14 @@
 (def ^:private users-name-field (delay (Field (id :users :name))))
 
 (def ^:private generic-sql-engines
-  (delay (set (for [engine datasets/all-valid-engines
-                    :let   [driver (driver/engine->driver engine)]
-                    :when  (not (contains? #{:bigquery :presto :sparksql} engine))       ; bigquery, presto and sparksql don't use the generic sql implementations of things like `field-avg-length`
-                    :when  (extends? ISQLDriver (class driver))]
-                (do (require (symbol (str "metabase.test.data." (name engine))) :reload) ; otherwise it gets all snippy if you try to do `lein test metabase.driver.generic-sql-test`
-                    engine)))))
+  (delay
+   (du/profile "resolve metabase.driver.generic-sql-test/generic-sql-engines"
+     (set (for [engine datasets/all-valid-engines
+                :let   [driver (driver/engine->driver engine)]
+                :when  (not (contains? #{:bigquery :presto :sparksql} engine)) ; bigquery, presto and sparksql don't use the generic sql implementations of things like `field-avg-length`
+                :when  (extends? ISQLDriver (class driver))]
+            (do (require (symbol (str "metabase.test.data." (name engine))) :reload) ; otherwise it gets all snippy if you try to do `lein test metabase.driver.generic-sql-test`
+                engine))))))
 
 
 ;; DESCRIBE-DATABASE

test/metabase/test/data.clj

@@ -286,11 +286,10 @@
          get-or-create! (fn []
                           (or (i/metabase-instance database-definition engine)
                               (create-database! database-definition engine driver)))]
+     ;; attempt to make sure test extensions are loaded for the driver. This might still fail (see below)
+     (require (symbol (str "metabase.test.data." (name engine))))
      (try
-       ;; it's ok to ignore output here because it's usually the IllegalArgException, and if it fails again we don't
-       ;; suppress it
-       (tu.log/suppress-output
-         (get-or-create!))
+       (get-or-create!)
        ;; occasionally we'll see an error like
        ;;   java.lang.IllegalArgumentException: No implementation of method: :database->connection-details
        ;;   of protocol: IDriverTestExtensions found for class: metabase.driver.h2.H2Driver
@@ -371,7 +370,7 @@
 (defmacro with-data [data-load-fn & body]
   `(call-with-data ~data-load-fn (fn [] ~@body)))
 
-(def venue-categories
+(def ^:private venue-categories
   (map vector (defs/field-values defs/test-data-map "categories" "name")))
 
 (defn create-venue-category-remapping

test/metabase/test/data/datasets.clj

 (ns metabase.test.data.datasets
   "Interface + implementations for loading test datasets for different drivers, and getting information about the
-  dataset's tables, fields, etc."
+  dataset's tables, fields, etc.
+
+  TODO - we should seriously rename this namespace to something like `metabase.test.driver` or something like that.
+  Also need to stop using 'engine' to mean 'driver keyword'."
   (:require [clojure.string :as s]
             [clojure.tools.logging :as log]
             [colorize.core :as color]
@@ -10,13 +13,16 @@
              [config :as config]
              [driver :as driver]
              [plugins :as plugins]]
-            [metabase.test.data.interface :as i]))
+            [metabase.test.data.interface :as i]
+            [metabase.util.date :as du]))
 
 ;; When running tests, we need to make sure plugins (i.e., the Oracle JDBC driver) are loaded because otherwise the
 ;; Oracle driver won't show up in the list of valid drivers below
-(plugins/load-plugins!)
+(du/profile "(plugins/load-plugins!) (in metabase.test.data.datasets)"
+  (plugins/load-plugins!))
 
-(driver/find-and-load-drivers!)
+(du/profile "(driver/find-and-load-drivers!) (in metabase.test.data.datasets)"
+  (driver/find-and-load-drivers!))
 
 (def ^:const all-valid-engines (set (keys (driver/available-drivers))))
 
@@ -159,8 +165,8 @@
   `(expect-with-engines all-valid-engines ~expected ~actual))
 
 
-;;; Load metabase.test.data.* namespaces for all available drivers
-(doseq [engine all-valid-engines]
-  (let [driver-test-namespace (engine->test-extensions-ns-symbol engine)]
-    (when (find-ns driver-test-namespace)
-      (require driver-test-namespace))))
+(du/profile "Load metabase.test.data.* namespaces for all available drivers"
+  (doseq [engine all-valid-engines]
+    (let [driver-test-namespace (engine->test-extensions-ns-symbol engine)]
+      (when (find-ns driver-test-namespace)
+        (require driver-test-namespace)))))

test/metabase/test_setup.clj

@@ -13,7 +13,8 @@
              [task :as task]
              [util :as u]]
             [metabase.core.initialization-status :as init-status]
-            [metabase.models.setting :as setting]))
+            [metabase.models.setting :as setting]
+            [metabase.util.date :as du]))
 
 ;;; ---------------------------------------- Expectations Framework Settings -----------------------------------------
 
@@ -63,7 +64,8 @@
 
 ;; `test-startup` function won't work for loading the drivers because they need to be available at evaluation time for
 ;; some of the unit tests work work properly
-(driver/find-and-load-drivers!)
+(du/profile "(driver/find-and-load-drivers!) (in metabase.test-setup)"
+  (driver/find-and-load-drivers!))
 
 (defn test-startup
   {:expectations-options :before-run}
@@ -85,10 +87,11 @@
 
       ;; make sure the driver test extensions are loaded before running the tests. :reload them because otherwise we
       ;; get wacky 'method in protocol not implemented' errors when running tests against an individual namespace
-      (doseq [engine (keys (driver/available-drivers))
-              :let   [driver-test-ns (symbol (str "metabase.test.data." (name engine)))]]
-        (u/ignore-exceptions
-          (require driver-test-ns :reload)))
+      (du/profile "Load all drivers & reload test extensions"
+        (doseq [engine (keys (driver/available-drivers))
+                :let   [driver-test-ns (symbol (str "metabase.test.data." (name engine)))]]
+          (u/ignore-exceptions
+            (require driver-test-ns :reload))))
 
       ;; If test setup fails exit right away
       (catch Throwable e