diff --git a/modules/drivers/druid/test/metabase/test/data/druid.clj b/modules/drivers/druid/test/metabase/test/data/druid.clj
index 94b7bfd2397f7598b5954d9f85ea9d6c07210a1d..177b269d2b4caa22bf5e1c5a7728470aaf02852e 100644
--- a/modules/drivers/druid/test/metabase/test/data/druid.clj
+++ b/modules/drivers/druid/test/metabase/test/data/druid.clj
@@ -30,13 +30,14 @@
;;; Generating Data File
(defn- flattened-test-data []
- (let [dbdef (tx/flatten-dbdef defs/test-data "checkins")
+ (let [dbdef (tx/get-dataset-definition
+ (tx/flattened-dataset-definition defs/test-data "checkins"))
tabledef (first (:table-definitions dbdef))]
(->> (:rows tabledef)
(map (partial zipmap (map :field-name (:field-definitions tabledef))))
(map-indexed (fn [i row]
(assoc row :id (inc i))))
- (sort-by (u/rpartial get "date")))))
+ (sort-by #(get % "date")))))
(defn- write-dbdef-to-json [db-def filename]
(io/delete-file filename :silently)
diff --git a/modules/drivers/mongo/test/metabase/driver/mongo_test.clj b/modules/drivers/mongo/test/metabase/driver/mongo_test.clj
index 1400797dc0c75a4cf77f83192d0a868dea20fb5f..20f8efc1d666dc477f9800d55879c6e06c6c1773 100644
--- a/modules/drivers/mongo/test/metabase/driver/mongo_test.clj
+++ b/modules/drivers/mongo/test/metabase/driver/mongo_test.clj
@@ -126,12 +126,12 @@
(driver/describe-table :mongo (data/db) (Table (data/id :venues))))
;; Make sure that all-NULL columns work and are synced correctly (#6875)
-(tx/def-database-definition ^:private all-null-columns
+(tx/defdataset ^:private all-null-columns
[["bird_species"
- [{:field-name "name", :base-type :type/Text}
- {:field-name "favorite_snack", :base-type :type/Text}]
- [["House Finch" nil]
- ["Mourning Dove" nil]]]])
+ [{:field-name "name", :base-type :type/Text}
+ {:field-name "favorite_snack", :base-type :type/Text}]
+ [["House Finch" nil]
+ ["Mourning Dove" nil]]]])
(datasets/expect-with-driver :mongo
[{:name "_id", :database_type "java.lang.Long", :base_type :type/Integer, :special_type :type/PK}
@@ -198,7 +198,7 @@
;;; Check that we support Mongo BSON ID and can filter by it (#1367)
-(tx/def-database-definition ^:private with-bson-ids
+(tx/defdataset ^:private with-bson-ids
[["birds"
[{:field-name "name", :base-type :type/Text}
{:field-name "bird_id", :base-type :type/MongoBSONID}]
diff --git a/modules/drivers/sqlserver/test/metabase/driver/sqlserver_test.clj b/modules/drivers/sqlserver/test/metabase/driver/sqlserver_test.clj
index 9a333cdeb1530cd6229ba85d1125bde8f6265b18..909024229f5f4da3b0735cfad8aff1aee3f55fdf 100644
--- a/modules/drivers/sqlserver/test/metabase/driver/sqlserver_test.clj
+++ b/modules/drivers/sqlserver/test/metabase/driver/sqlserver_test.clj
@@ -16,16 +16,16 @@
[util :as tu :refer [obj->json->obj]]]
[metabase.test.data
[datasets :as datasets]
- [interface :as tx :refer [def-database-definition]]]))
+ [interface :as tx]]))
;;; -------------------------------------------------- VARCHAR(MAX) --------------------------------------------------
;; Make sure something long doesn't come back as some weird type like `ClobImpl`
-(def ^:private ^:const a-gene
+(def ^:private a-gene
"Really long string representing a gene like \"GGAGCACCTCCACAAGTGCAGGCTATCCTGTCGAGTAAGGCCT...\""
(apply str (repeatedly 1000 (partial rand-nth [\A \G \C \T]))))
-(def-database-definition ^:private ^:const genetic-data
+(tx/defdataset ^:private genetic-data
[["genetic-data"
[{:field-name "gene", :base-type {:native "VARCHAR(MAX)"}}]
[[a-gene]]]])
diff --git a/src/metabase/db.clj b/src/metabase/db.clj
index b6de5e05e4623a7c2cea13e61c251c7298e42797..d820205527b91d1fea656d640957cfb29644a759 100644
--- a/src/metabase/db.clj
+++ b/src/metabase/db.clj
@@ -466,7 +466,7 @@
(u/with-us-locale
(verify-db-connection db-details)
(run-schema-migrations! auto-migrate db-details)
- (create-connection-pool! (du/profile (jdbc-details db-details)))
+ (create-connection-pool! (jdbc-details db-details))
(run-data-migrations!)
(reset! db-setup-finished? true))))
nil)
diff --git a/src/metabase/query_processor.clj b/src/metabase/query_processor.clj
index 0a066fe87c80782bbc557423a88eedda53164cc8..a33eb1f99113ad4c094ce2fdb33546708cc302b3 100644
--- a/src/metabase/query_processor.clj
+++ b/src/metabase/query_processor.clj
@@ -233,9 +233,11 @@
(def ^:private default-pipeline (qp-pipeline execute-query))
(def ^:private QueryResponse
- (s/cond-pre
- ManyToManyChannel
- {:status (s/enum :completed :failed :canceled), s/Any s/Any}))
+ (s/named
+ (s/cond-pre
+ ManyToManyChannel
+ {:status (s/enum :completed :failed :canceled), s/Any s/Any})
+ "Valid query response (core.async channel or map with :status)"))
(s/defn process-query :- QueryResponse
"Process an MBQL query. This is the main entrypoint to the magical realm of the Query Processor. Returns a
diff --git a/test/metabase/api/dataset_test.clj b/test/metabase/api/dataset_test.clj
index 0864353fddc25fb7b6d5f0d7b2bb9cf49fda3f49..70c44398441938a634acc59b07bfb40d551c65c8 100644
--- a/test/metabase/api/dataset_test.clj
+++ b/test/metabase/api/dataset_test.clj
@@ -203,7 +203,7 @@
["3" "2014-09-15" "" "8" "56"]
["4" "2014-03-11" "" "5" "4"]
["5" "2013-05-05" "" "3" "49"]]
- (data/with-db (data/get-or-create-database! defs/test-data-with-null-date-checkins)
+ (data/dataset defs/test-data-with-null-date-checkins
(let [result ((test-users/user->client :rasta) :post 200 "dataset/csv" :query
(json/generate-string (data/mbql-query checkins)))]
(take 5 (parse-and-sort-csv result)))))
diff --git a/test/metabase/api/table_test.clj b/test/metabase/api/table_test.clj
index f898aea91f8f4491aea929778067ab0ebd3679da..7726c73d257692682408909472fd48c99d9d96a1 100644
--- a/test/metabase/api/table_test.clj
+++ b/test/metabase/api/table_test.clj
@@ -23,7 +23,6 @@
[data :as data]
[util :as tu :refer [match-$]]]
[metabase.test.data
- [dataset-definitions :as defs]
[datasets :as datasets]
[users :refer [user->client]]]
[metabase.test.mock.util :as mutil]
@@ -205,24 +204,6 @@
:fields_hash $}))
((user->client :rasta) :get 200 (format "table/%d/query_metadata" (data/id :categories))))
-
-(def ^:private user-last-login-date-strs
- "In an effort to be really annoying, the date strings returned by the API are different on Circle than they are
- locally. Generate strings like '2014-01-01' at runtime so we get matching values."
- (let [format-inst (fn [^java.util.Date inst]
- (format "%d-%02d-%02d"
- (+ (.getYear inst) 1900)
- (+ (.getMonth inst) 1)
- (.getDate inst)))]
- (->> (defs/field-values defs/test-data-map "users" "last_login")
- (map format-inst)
- set
- sort
- vec)))
-
-(def ^:private user-full-names
- (defs/field-values defs/test-data-map "users" "name"))
-
;;; GET api/table/:id/query_metadata?include_sensitive_fields
;; Make sure that getting the User table *does* include info about the password field, but not actual values
;; themselves
diff --git a/test/metabase/driver/mysql_test.clj b/test/metabase/driver/mysql_test.clj
index 4c51b3c6aba44d14f89775a17f565f2bdfe68b9b..4c6a3132cd2dbedd23503d17d17a35e92183c6ce 100644
--- a/test/metabase/driver/mysql_test.clj
+++ b/test/metabase/driver/mysql_test.clj
@@ -17,7 +17,7 @@
[util :as tu]]
[metabase.test.data
[datasets :refer [expect-with-driver]]
- [interface :as tx :refer [def-database-definition]]]
+ [interface :as tx]]
[metabase.test.util.timezone :as tu.tz]
[metabase.util.date :as du]
[toucan.db :as db]
@@ -54,7 +54,7 @@
;; Test how TINYINT(1) columns are interpreted. By default, they should be interpreted as integers, but with the
;; correct additional options, we should be able to change that -- see
;; https://github.com/metabase/metabase/issues/3506
-(def-database-definition ^:private ^:const tiny-int-ones
+(tx/defdataset ^:private tiny-int-ones
[["number-of-cans"
[{:field-name "thing", :base-type :type/Text}
{:field-name "number-of-cans", :base-type {:native "tinyint(1)"}}]
@@ -71,7 +71,7 @@
#{{:name "number-of-cans", :base_type :type/Boolean, :special_type :type/Category}
{:name "id", :base_type :type/Integer, :special_type :type/PK}
{:name "thing", :base_type :type/Text, :special_type :type/Category}}
- (data/with-temp-db [db tiny-int-ones]
+ (data/with-db-for-dataset [db tiny-int-ones]
(db->fields db)))
;; if someone says specifies `tinyInt1isBit=false`, it should come back as a number instead
@@ -79,7 +79,7 @@
#{{:name "number-of-cans", :base_type :type/Integer, :special_type :type/Quantity}
{:name "id", :base_type :type/Integer, :special_type :type/PK}
{:name "thing", :base_type :type/Text, :special_type :type/Category}}
- (data/with-temp-db [db tiny-int-ones]
+ (data/with-db-for-dataset [db tiny-int-ones]
(tt/with-temp Database [db {:engine "mysql"
:details (assoc (:details db)
:additional-options "tinyInt1isBit=false")}]
diff --git a/test/metabase/driver/postgres_test.clj b/test/metabase/driver/postgres_test.clj
index 138666efa1f2c637ffda14e75403d0832eccc9b0..00a617d8029335ee984140b119723dff4e74d156 100644
--- a/test/metabase/driver/postgres_test.clj
+++ b/test/metabase/driver/postgres_test.clj
@@ -64,9 +64,9 @@
;; Verify that we identify JSON columns and mark metadata properly during sync
(datasets/expect-with-driver :postgres
:type/SerializedJSON
- (data/with-temp-db
+ (data/with-db-for-dataset
[_
- (tx/create-database-definition "Postgres with a JSON Field"
+ (tx/dataset-definition "Postgres with a JSON Field"
["venues"
[{:field-name "address", :base-type {:native "json"}}]
[[(hsql/raw "to_json('{\"street\": \"431 Natoma\", \"city\": \"San Francisco\", \"state\": \"CA\", \"zip\": 94103}'::text)")]]])]
@@ -74,7 +74,7 @@
;;; # UUID Support
-(tx/def-database-definition ^:private with-uuid
+(tx/defdataset ^:private with-uuid
[["users"
[{:field-name "user_id", :base-type :type/UUID}]
[[#uuid "4f01dcfd-13f7-430c-8e6f-e505c0851027"]
@@ -125,7 +125,7 @@
;; Make sure that Tables / Fields with dots in their names get escaped properly
-(tx/def-database-definition ^:private dots-in-names
+(tx/defdataset ^:private dots-in-names
[["objects.stuff"
[{:field-name "dotted.name", :base-type :type/Text}]
[["toucan_cage"]
@@ -143,7 +143,7 @@
;; Make sure that duplicate column names (e.g. caused by using a FK) still return both columns
-(tx/def-database-definition ^:private duplicate-names
+(tx/defdataset ^:private duplicate-names
[["birds"
[{:field-name "name", :base-type :type/Text}]
[["Rasta"]
@@ -163,7 +163,7 @@
;;; Check support for `inet` columns
-(tx/def-database-definition ^:private ip-addresses
+(tx/defdataset ^:private ip-addresses
[["addresses"
[{:field-name "ip", :base-type {:native "inet"}}]
[[(hsql/raw "'192.168.1.1'::inet")]
diff --git a/test/metabase/query_processor/async_test.clj b/test/metabase/query_processor/async_test.clj
index 85247ea865b145d5cec6f7fd327e4c7cf1be1f8e..4c483ba6bdc1aaa33fa61b86b4613046432b5ce1 100644
--- a/test/metabase/query_processor/async_test.clj
+++ b/test/metabase/query_processor/async_test.clj
@@ -9,20 +9,21 @@
;; running a query async should give you the same results as running that query synchronously
(let [query
- {:database (data/id)
- :type :query
- :query {:source-table (data/id :venues)
- :fields [[:field-id (data/id :venues :name)]]
- :limit 5}}
+ (delay
+ {:database (data/id)
+ :type :query
+ :query {:source-table (data/id :venues)
+ :fields [[:field-id (data/id :venues :name)]]
+ :limit 5}})
;; Metadata checksum might be encrypted if a encryption key is set on this system (to make it hard for bad
;; actors to forge one) in which case the checksums won't be equal.
maybe-decrypt-checksum
#(some-> % (update-in [:data :results_metadata :checksum] encrypt/maybe-decrypt))]
(expect
(maybe-decrypt-checksum
- (qp/process-query query))
+ (qp/process-query @query))
(maybe-decrypt-checksum
- (tu.async/with-open-channels [result-chan (qp.async/process-query query)]
+ (tu.async/with-open-channels [result-chan (qp.async/process-query @query)]
(first (a/alts!! [result-chan (a/timeout 1000)]))))))
(expect
diff --git a/test/metabase/query_processor_test.clj b/test/metabase/query_processor_test.clj
index 68e5dee50a7295dde22d22c0e2b687c92c3c43a4..1eba17927a4e18f44b9546e0be85dbd5e634e823 100644
--- a/test/metabase/query_processor_test.clj
+++ b/test/metabase/query_processor_test.clj
@@ -335,7 +335,8 @@
[results]
(when (= (:status results) :failed)
(println "Error running query:" (u/pprint-to-str 'red results))
- (throw (ex-info (:error results) results)))
+ (throw (ex-info (str (or (:error results) "Error running query"))
+ (if (map? results) results {:results results}))))
(:data results))
(defn rows
diff --git a/test/metabase/query_processor_test/breakout_test.clj b/test/metabase/query_processor_test/breakout_test.clj
index d0f270ec2e46a45a88b5d5154fc049b43130bb95..0cf331a834d9dda5dce99c6e34e7ea4c5e3e7623 100644
--- a/test/metabase/query_processor_test/breakout_test.clj
+++ b/test/metabase/query_processor_test/breakout_test.clj
@@ -14,9 +14,7 @@
[metabase.test
[data :as data]
[util :as tu]]
- [metabase.test.data
- [dataset-definitions :as defs]
- [datasets :as datasets]]
+ [metabase.test.data.datasets :as datasets]
[metabase.test.util.log :as tu.log]
[toucan.db :as db]
[toucan.util.test :as tt]))
@@ -107,7 +105,7 @@
:native_form true}
(data/with-data
(fn []
- (let [venue-names (defs/field-values defs/test-data-map "categories" "name")]
+ (let [venue-names (data/dataset-field-values "categories" "name")]
[(db/insert! Dimension {:field_id (data/id :venues :category_id)
:name "Foo"
:type :internal})
diff --git a/test/metabase/query_processor_test/date_bucketing_test.clj b/test/metabase/query_processor_test/date_bucketing_test.clj
index a9f54e23742e4a639587f3f0914ac88b57e38a13..b6089c640b9c54383691bf906bd176aea751fc37 100644
--- a/test/metabase/query_processor_test/date_bucketing_test.clj
+++ b/test/metabase/query_processor_test/date_bucketing_test.clj
@@ -19,7 +19,7 @@
[format :as tformat]]
[metabase
[driver :as driver]
- [query-processor-test :refer :all]
+ [query-processor-test :as qp.test :refer :all]
[util :as u]]
[metabase.test
[data :as data]
@@ -30,7 +30,7 @@
[interface :as tx]]
[metabase.test.util.timezone :as tu.tz]
[metabase.util.date :as du])
- (:import org.joda.time.DateTime))
+ (:import [org.joda.time DateTime DateTimeZone]))
(defn- ->long-if-number [x]
(if (number? x)
@@ -54,8 +54,8 @@
{:aggregation [[:count]]
:breakout [[:datetime-field $timestamp unit]]
:limit 10}))
- rows (format-rows-by [->long-if-number int])))
- ([unit tz]
+ rows (qp.test/format-rows-by [->long-if-number int])))
+ ([unit, ^DateTimeZone tz]
(tu/with-temporary-setting-values [report-timezone (.getID tz)]
(sad-toucan-incidents-with-bucketing unit))))
@@ -124,7 +124,7 @@
(repeat 1)))
;; Bucket sad toucan events by their default bucketing, which is the full datetime value
-(expect-with-non-timeseries-dbs
+(qp.test/expect-with-non-timeseries-dbs
(cond
;; Timezone is omitted by these databases
(= :sqlite driver/*driver*)
@@ -145,7 +145,7 @@
(sad-toucan-incidents-with-bucketing :default pacific-tz))
;; Buckets sad toucan events like above, but uses the eastern timezone as the report timezone
-(expect-with-non-timeseries-dbs
+(qp.test/expect-with-non-timeseries-dbs
(cond
;; These databases are always in UTC so aren't impacted by changes in report-timezone
(= :sqlite driver/*driver*)
@@ -170,7 +170,7 @@
;;
;; The exclusions here are databases that give incorrect answers when the JVM timezone doesn't match the databases
;; timezone
-(expect-with-non-timeseries-dbs-except #{:h2 :sqlserver :redshift :sparksql :mongo}
+(qp.test/expect-with-non-timeseries-dbs-except #{:h2 :sqlserver :redshift :sparksql :mongo}
(cond
(= :sqlite driver/*driver*)
(sad-toucan-result (source-date-formatter utc-tz) result-date-formatter-without-tz)
@@ -195,7 +195,7 @@
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; This dataset doesn't have multiple events in a minute, the results are the same as the default grouping
-(expect-with-non-timeseries-dbs
+(qp.test/expect-with-non-timeseries-dbs
(cond
(= :sqlite driver/*driver*)
(sad-toucan-result (source-date-formatter utc-tz) result-date-formatter-without-tz)
@@ -211,7 +211,7 @@
(sad-toucan-incidents-with-bucketing :minute pacific-tz))
;; Grouping by minute of hour is not affected by timezones
-(expect-with-non-timeseries-dbs
+(qp.test/expect-with-non-timeseries-dbs
[[0 5]
[1 4]
[2 2]
@@ -258,7 +258,7 @@
;; formatting of the time for that given count is different depending
;; on whether the database supports a report timezone and what
;; timezone that database is in
-(expect-with-non-timeseries-dbs
+(qp.test/expect-with-non-timeseries-dbs
(cond
(= :sqlite driver/*driver*)
(results-by-hour (source-date-formatter utc-tz)
@@ -285,7 +285,7 @@
;; by 7 hours. These count changes can be validated by matching the
;; first three results of the pacific results to the last three of the
;; UTC results (i.e. pacific is 7 hours back of UTC at that time)
-(expect-with-non-timeseries-dbs
+(qp.test/expect-with-non-timeseries-dbs
(if (and (not (tz-shifted-engine-bug? driver/*driver*))
(supports-report-timezone? driver/*driver*))
[[0 8] [1 9] [2 7] [3 10] [4 10] [5 9] [6 6] [7 5] [8 7] [9 7]]
@@ -293,7 +293,7 @@
(sad-toucan-incidents-with-bucketing :hour-of-day pacific-tz))
;; With all databases in UTC, the results should be the same for all DBs
-(expect-with-non-timeseries-dbs
+(qp.test/expect-with-non-timeseries-dbs
[[0 13] [1 8] [2 4] [3 7] [4 5] [5 13] [6 10] [7 8] [8 9] [9 7]]
(sad-toucan-incidents-with-bucketing :hour-of-day utc-tz))
@@ -305,7 +305,7 @@
(defn- offset-time
"Add to `date` offset from UTC found in `tz`"
- [tz date]
+ [^DateTimeZone tz, ^DateTime date]
(time/minus date
(time/seconds
(/ (.getOffset tz date) 1000))))
@@ -372,7 +372,7 @@
(adjust-date source-formatter result-formatter sad-toucan-events-grouped-by-day)
counts))
-(expect-with-non-timeseries-dbs
+(qp.test/expect-with-non-timeseries-dbs
(if (= :sqlite driver/*driver*)
(results-by-day date-formatter-without-time
date-formatter-without-time
@@ -383,7 +383,7 @@
(sad-toucan-incidents-with-bucketing :day utc-tz))
-(expect-with-non-timeseries-dbs
+(qp.test/expect-with-non-timeseries-dbs
(cond
(= :sqlite driver/*driver*)
(results-by-day date-formatter-without-time
@@ -416,7 +416,7 @@
["01" "02" "03" "04" "05" "06" "07" "08" "09" "10"]))
;; Similar to the pacific test above, just validating eastern timezone shifts
-(expect-with-non-timeseries-dbs
+(qp.test/expect-with-non-timeseries-dbs
(cond
(= :sqlite driver/*driver*)
(results-by-day date-formatter-without-time
@@ -449,7 +449,7 @@
;;
;; The exclusions here are databases that give incorrect answers when the JVM timezone doesn't match the databases
;; timezone
-(expect-with-non-timeseries-dbs-except #{:h2 :sqlserver :redshift :sparksql :mongo}
+(qp.test/expect-with-non-timeseries-dbs-except #{:h2 :sqlserver :redshift :sparksql :mongo}
(cond
(= :sqlite driver/*driver*)
(results-by-day date-formatter-without-time
@@ -480,14 +480,14 @@
;;
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-(expect-with-non-timeseries-dbs
+(qp.test/expect-with-non-timeseries-dbs
(if (and (not (tz-shifted-engine-bug? driver/*driver*))
(supports-report-timezone? driver/*driver*))
[[1 29] [2 36] [3 33] [4 29] [5 13] [6 38] [7 22]]
[[1 28] [2 38] [3 29] [4 27] [5 24] [6 30] [7 24]])
(sad-toucan-incidents-with-bucketing :day-of-week pacific-tz))
-(expect-with-non-timeseries-dbs
+(qp.test/expect-with-non-timeseries-dbs
[[1 28] [2 38] [3 29] [4 27] [5 24] [6 30] [7 24]]
(sad-toucan-incidents-with-bucketing :day-of-week utc-tz))
@@ -497,14 +497,14 @@
;;
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-(expect-with-non-timeseries-dbs
+(qp.test/expect-with-non-timeseries-dbs
(if (and (not (tz-shifted-engine-bug? driver/*driver*))
(supports-report-timezone? driver/*driver*))
[[1 8] [2 9] [3 9] [4 4] [5 11] [6 8] [7 6] [8 10] [9 6] [10 10]]
[[1 6] [2 10] [3 4] [4 9] [5 9] [6 8] [7 8] [8 9] [9 7] [10 9]])
(sad-toucan-incidents-with-bucketing :day-of-month pacific-tz))
-(expect-with-non-timeseries-dbs
+(qp.test/expect-with-non-timeseries-dbs
[[1 6] [2 10] [3 4] [4 9] [5 9] [6 8] [7 8] [8 9] [9 7] [10 9]]
(sad-toucan-incidents-with-bucketing :day-of-month utc-tz))
@@ -514,14 +514,14 @@
;;
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-(expect-with-non-timeseries-dbs
+(qp.test/expect-with-non-timeseries-dbs
(if (and (not (tz-shifted-engine-bug? driver/*driver*))
(supports-report-timezone? driver/*driver*))
[[152 8] [153 9] [154 9] [155 4] [156 11] [157 8] [158 6] [159 10] [160 6] [161 10]]
[[152 6] [153 10] [154 4] [155 9] [156 9] [157 8] [158 8] [159 9] [160 7] [161 9]])
(sad-toucan-incidents-with-bucketing :day-of-year pacific-tz))
-(expect-with-non-timeseries-dbs
+(qp.test/expect-with-non-timeseries-dbs
[[152 6] [153 10] [154 4] [155 9] [156 9] [157 8] [158 8] [159 9] [160 7] [161 9]]
(sad-toucan-incidents-with-bucketing :day-of-year utc-tz))
@@ -543,7 +543,7 @@
"2015-06-28"])
counts))
-(expect-with-non-timeseries-dbs
+(qp.test/expect-with-non-timeseries-dbs
(if (= :sqlite driver/*driver*)
(results-by-week date-formatter-without-time
date-formatter-without-time
@@ -578,7 +578,7 @@
;; Sad toucan incidents by week. Databases in UTC that don't support report timezones will be the same as the UTC test
;; above. Databases that support report timezone will have different counts as the week starts and ends 7 hours
;; earlier
-(expect-with-non-timeseries-dbs
+(qp.test/expect-with-non-timeseries-dbs
(cond
(= :sqlite driver/*driver*)
(results-by-week date-formatter-without-time
@@ -610,7 +610,7 @@
;; Tests eastern timezone grouping by week, UTC databases don't change, databases with reporting timezones need to
;; account for the 4-5 hour difference
-(expect-with-non-timeseries-dbs
+(qp.test/expect-with-non-timeseries-dbs
(cond
(= :sqlite driver/*driver*)
(results-by-week date-formatter-without-time
@@ -639,7 +639,7 @@
;;
;; The exclusions here are databases that give incorrect answers when the JVM timezone doesn't match the databases
;; timezone
-(expect-with-non-timeseries-dbs-except #{:h2 :sqlserver :redshift :sparksql :mongo}
+(qp.test/expect-with-non-timeseries-dbs-except #{:h2 :sqlserver :redshift :sparksql :mongo}
(cond
(= :sqlite driver/*driver*)
(results-by-week date-formatter-without-time
@@ -669,7 +669,7 @@
;;
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-(expect-with-non-timeseries-dbs
+(qp.test/expect-with-non-timeseries-dbs
;; Not really sure why different drivers have different opinions on these </3
(cond
(= :snowflake driver/*driver*)
@@ -694,7 +694,7 @@
;; All of the sad toucan events in the test data fit in June. The results are the same on all databases and the only
;; difference is how the beginning of hte month is represented, since we always return times with our dates
-(expect-with-non-timeseries-dbs
+(qp.test/expect-with-non-timeseries-dbs
[[(cond
(= :sqlite driver/*driver*)
"2015-06-01"
@@ -707,7 +707,7 @@
200]]
(sad-toucan-incidents-with-bucketing :month pacific-tz))
-(expect-with-non-timeseries-dbs
+(qp.test/expect-with-non-timeseries-dbs
[[(cond
(= :sqlite driver/*driver*)
"2015-06-01"
@@ -726,7 +726,7 @@
;;
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-(expect-with-non-timeseries-dbs
+(qp.test/expect-with-non-timeseries-dbs
[[6 200]]
(sad-toucan-incidents-with-bucketing :month-of-year pacific-tz))
@@ -736,7 +736,7 @@
;;
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-(expect-with-non-timeseries-dbs
+(qp.test/expect-with-non-timeseries-dbs
[[(cond (= :sqlite driver/*driver*)
"2015-04-01"
@@ -748,7 +748,7 @@
200]]
(sad-toucan-incidents-with-bucketing :quarter pacific-tz))
-(expect-with-non-timeseries-dbs
+(qp.test/expect-with-non-timeseries-dbs
[[(cond (= :sqlite driver/*driver*)
"2015-04-01"
@@ -766,7 +766,7 @@
;;
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-(expect-with-non-timeseries-dbs
+(qp.test/expect-with-non-timeseries-dbs
[[2 200]]
(sad-toucan-incidents-with-bucketing :quarter-of-year pacific-tz))
@@ -776,7 +776,7 @@
;;
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-(expect-with-non-timeseries-dbs
+(qp.test/expect-with-non-timeseries-dbs
[[2015 200]]
(sad-toucan-incidents-with-bucketing :year pacific-tz))
@@ -787,30 +787,37 @@
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; RELATIVE DATES
-(defn- database-def-with-timestamps [interval-seconds]
- (tx/create-database-definition (str "a-checkin-every-" interval-seconds "-seconds")
- ["checkins"
- [{:field-name "timestamp"
- :base-type :type/DateTime}]
- (vec (for [i (range -15 15)]
- ;; Create timestamps using relative dates (e.g. `DATEADD(second, -195, GETUTCDATE())` instead of
- ;; generating `java.sql.Timestamps` here so they'll be in the DB's native timezone. Some DBs refuse to use
- ;; the same timezone we're running the tests from *cough* SQL Server *cough*
- [(u/prog1 (driver/date-interval driver/*driver* :second (* i interval-seconds))
- (assert <>))]))]))
-
-(def ^:private checkins:4-per-minute (partial database-def-with-timestamps 15))
-(def ^:private checkins:4-per-hour (partial database-def-with-timestamps (* 60 15)))
-(def ^:private checkins:1-per-day (partial database-def-with-timestamps (* 60 60 24)))
-
-(defn- count-of-grouping [db field-grouping & relative-datetime-args]
- (-> (data/with-temp-db [_ db]
+(deftype ^:private TimestampDatasetDef [intervalSeconds])
+
+(defmethod tx/get-dataset-definition TimestampDatasetDef
+ [^TimestampDatasetDef this]
+ (let [interval-seconds (.intervalSeconds this)]
+ (tx/dataset-definition (str "checkins_interval_" interval-seconds)
+ ["checkins"
+ [{:field-name "timestamp"
+ :base-type :type/DateTime}]
+ (vec (for [i (range -15 15)]
+ ;; Create timestamps using relative dates (e.g. `DATEADD(second, -195, GETUTCDATE())` instead of
+ ;; generating `java.sql.Timestamps` here so they'll be in the DB's native timezone. Some DBs refuse to use
+ ;; the same timezone we're running the tests from *cough* SQL Server *cough*
+ [(u/prog1 (driver/date-interval driver/*driver* :second (* i interval-seconds))
+ (assert <>))]))])))
+
+(defn- dataset-def-with-timestamps [interval-seconds]
+ (TimestampDatasetDef. interval-seconds))
+
+(def ^:private checkins:4-per-minute (dataset-def-with-timestamps 15))
+(def ^:private checkins:4-per-hour (dataset-def-with-timestamps (* 60 15)))
+(def ^:private checkins:1-per-day (dataset-def-with-timestamps (* 60 60 24)))
+
+(defn- count-of-grouping [dataset field-grouping & relative-datetime-args]
+ (-> (data/with-db-for-dataset [_ dataset]
(data/run-mbql-query checkins
{:aggregation [[:count]]
:filter [:=
[:datetime-field $timestamp field-grouping]
(cons :relative-datetime relative-datetime-args)]}))
- first-row first int))
+ qp.test/first-row first int))
;; HACK - Don't run these tests against BigQuery/etc. because the databases need to be loaded every time the tests are ran
;; and loading data into BigQuery/etc. is mind-bogglingly slow. Don't worry, I promise these work though!
@@ -818,44 +825,44 @@
;; Don't run the minute tests against Oracle because the Oracle tests are kind of slow and case CI to fail randomly
;; when it takes so long to load the data that the times are no longer current (these tests pass locally if your
;; machine isn't as slow as the CircleCI ones)
-(expect-with-non-timeseries-dbs-except #{:snowflake :bigquery :oracle} 4 (count-of-grouping (checkins:4-per-minute) :minute "current"))
+(qp.test/expect-with-non-timeseries-dbs-except #{:snowflake :bigquery :oracle} 4 (count-of-grouping checkins:4-per-minute :minute "current"))
-(expect-with-non-timeseries-dbs-except #{:snowflake :bigquery :oracle} 4 (count-of-grouping (checkins:4-per-minute) :minute -1 "minute"))
-(expect-with-non-timeseries-dbs-except #{:snowflake :bigquery :oracle} 4 (count-of-grouping (checkins:4-per-minute) :minute 1 "minute"))
+(qp.test/expect-with-non-timeseries-dbs-except #{:snowflake :bigquery :oracle} 4 (count-of-grouping checkins:4-per-minute :minute -1 "minute"))
+(qp.test/expect-with-non-timeseries-dbs-except #{:snowflake :bigquery :oracle} 4 (count-of-grouping checkins:4-per-minute :minute 1 "minute"))
-(expect-with-non-timeseries-dbs-except #{:snowflake :bigquery} 4 (count-of-grouping (checkins:4-per-hour) :hour "current"))
-(expect-with-non-timeseries-dbs-except #{:snowflake :bigquery} 4 (count-of-grouping (checkins:4-per-hour) :hour -1 "hour"))
-(expect-with-non-timeseries-dbs-except #{:snowflake :bigquery} 4 (count-of-grouping (checkins:4-per-hour) :hour 1 "hour"))
+(qp.test/expect-with-non-timeseries-dbs-except #{:snowflake :bigquery} 4 (count-of-grouping checkins:4-per-hour :hour "current"))
+(qp.test/expect-with-non-timeseries-dbs-except #{:snowflake :bigquery} 4 (count-of-grouping checkins:4-per-hour :hour -1 "hour"))
+(qp.test/expect-with-non-timeseries-dbs-except #{:snowflake :bigquery} 4 (count-of-grouping checkins:4-per-hour :hour 1 "hour"))
-(expect-with-non-timeseries-dbs-except #{:snowflake :bigquery} 1 (count-of-grouping (checkins:1-per-day) :day "current"))
-(expect-with-non-timeseries-dbs-except #{:snowflake :bigquery} 1 (count-of-grouping (checkins:1-per-day) :day -1 "day"))
-(expect-with-non-timeseries-dbs-except #{:snowflake :bigquery} 1 (count-of-grouping (checkins:1-per-day) :day 1 "day"))
+(qp.test/expect-with-non-timeseries-dbs-except #{:snowflake :bigquery} 1 (count-of-grouping checkins:1-per-day :day "current"))
+(qp.test/expect-with-non-timeseries-dbs-except #{:snowflake :bigquery} 1 (count-of-grouping checkins:1-per-day :day -1 "day"))
+(qp.test/expect-with-non-timeseries-dbs-except #{:snowflake :bigquery} 1 (count-of-grouping checkins:1-per-day :day 1 "day"))
-(expect-with-non-timeseries-dbs-except #{:snowflake :bigquery} 7 (count-of-grouping (checkins:1-per-day) :week "current"))
+(qp.test/expect-with-non-timeseries-dbs-except #{:snowflake :bigquery} 7 (count-of-grouping checkins:1-per-day :week "current"))
;; SYNTACTIC SUGAR
-(expect-with-non-timeseries-dbs-except #{:snowflake :bigquery}
+(qp.test/expect-with-non-timeseries-dbs-except #{:snowflake :bigquery}
1
- (-> (data/with-temp-db [_ (checkins:1-per-day)]
+ (-> (data/with-db-for-dataset [_ checkins:1-per-day]
(data/run-mbql-query checkins
{:aggregation [[:count]]
:filter [:time-interval $timestamp :current :day]}))
- first-row first int))
+ qp.test/first-row first int))
-(expect-with-non-timeseries-dbs-except #{:snowflake :bigquery}
+(qp.test/expect-with-non-timeseries-dbs-except #{:snowflake :bigquery}
7
- (-> (data/with-temp-db [_ (checkins:1-per-day)]
+ (-> (data/with-db-for-dataset [_ checkins:1-per-day]
(data/run-mbql-query checkins
{:aggregation [[:count]]
:filter [:time-interval $timestamp :last :week]}))
- first-row first int))
+ qp.test/first-row first int))
;; Make sure that when referencing the same field multiple times with different units we return the one that actually
;; reflects the units the results are in. eg when we breakout by one unit and filter by another, make sure the results
;; and the col info use the unit used by breakout
(defn- date-bucketing-unit-when-you [& {:keys [breakout-by filter-by with-interval]
:or {with-interval :current}}]
- (let [results (data/with-temp-db [_ (checkins:1-per-day)]
+ (let [results (data/with-db-for-dataset [_ checkins:1-per-day]
(data/run-mbql-query checkins
{:aggregation [[:count]]
:breakout [[:datetime-field $timestamp breakout-by]]
@@ -864,33 +871,33 @@
(throw (ex-info "Query failed!" results)))
:unit (-> results :data :cols first :unit)}))
-(expect-with-non-timeseries-dbs-except #{:snowflake :bigquery}
+(qp.test/expect-with-non-timeseries-dbs-except #{:snowflake :bigquery}
{:rows 1, :unit :day}
(date-bucketing-unit-when-you :breakout-by "day", :filter-by "day"))
-(expect-with-non-timeseries-dbs-except #{:snowflake :bigquery}
+(qp.test/expect-with-non-timeseries-dbs-except #{:snowflake :bigquery}
{:rows 7, :unit :day}
(date-bucketing-unit-when-you :breakout-by "day", :filter-by "week"))
-(expect-with-non-timeseries-dbs-except #{:snowflake :bigquery}
+(qp.test/expect-with-non-timeseries-dbs-except #{:snowflake :bigquery}
{:rows 1, :unit :week}
(date-bucketing-unit-when-you :breakout-by "week", :filter-by "day"))
-(expect-with-non-timeseries-dbs-except #{:snowflake :bigquery}
+(qp.test/expect-with-non-timeseries-dbs-except #{:snowflake :bigquery}
{:rows 1, :unit :quarter}
(date-bucketing-unit-when-you :breakout-by "quarter", :filter-by "day"))
-(expect-with-non-timeseries-dbs-except #{:snowflake :bigquery}
+(qp.test/expect-with-non-timeseries-dbs-except #{:snowflake :bigquery}
{:rows 1, :unit :hour}
(date-bucketing-unit-when-you :breakout-by "hour", :filter-by "day"))
;; make sure if you use a relative date bucket in the past (e.g. "past 2 months") you get the correct amount of rows
;; (#3910)
-(expect-with-non-timeseries-dbs-except #{:snowflake :bigquery}
+(qp.test/expect-with-non-timeseries-dbs-except #{:snowflake :bigquery}
{:rows 2, :unit :day}
(date-bucketing-unit-when-you :breakout-by "day", :filter-by "day", :with-interval -2))
-(expect-with-non-timeseries-dbs-except #{:snowflake :bigquery}
+(qp.test/expect-with-non-timeseries-dbs-except #{:snowflake :bigquery}
{:rows 2, :unit :day}
(date-bucketing-unit-when-you :breakout-by "day", :filter-by "day", :with-interval 2))
@@ -904,31 +911,31 @@
;;
;; We should get count = 1 for the current day, as opposed to count = 0 if we weren't auto-bucketing
;; (e.g. 2018-11-19T00:00 != 2018-11-19T12:37 or whatever time the checkin is at)
-(expect-with-non-timeseries-dbs-except #{:snowflake :bigquery}
+(qp.test/expect-with-non-timeseries-dbs-except #{:snowflake :bigquery}
[[1]]
- (format-rows-by [int]
- (rows
- (data/with-temp-db [_ (checkins:1-per-day)]
+ (qp.test/format-rows-by [int]
+ (qp.test/rows
+ (data/with-db-for-dataset [_ checkins:1-per-day]
(data/run-mbql-query checkins
{:aggregation [[:count]]
:filter [:= [:field-id $timestamp] (du/format-date "yyyy-MM-dd" (du/date-trunc :day))]})))))
;; this is basically the same test as above, but using the office-checkins dataset instead of the dynamically created
;; checkins DBs so we can run it against Snowflake and BigQuery as well.
-(expect-with-non-timeseries-dbs
+(qp.test/expect-with-non-timeseries-dbs
[[1]]
- (format-rows-by [int]
- (rows
+ (qp.test/format-rows-by [int]
+ (qp.test/rows
(data/dataset office-checkins
(data/run-mbql-query checkins
{:aggregation [[:count]]
:filter [:= [:field-id $timestamp] "2019-01-16"]})))))
;; Check that automatic bucketing still happens when using compound filter clauses (#9127)
-(expect-with-non-timeseries-dbs
+(qp.test/expect-with-non-timeseries-dbs
[[1]]
- (format-rows-by [int]
- (rows
+ (qp.test/format-rows-by [int]
+ (qp.test/rows
(data/dataset office-checkins
(data/run-mbql-query checkins
{:aggregation [[:count]]
@@ -937,14 +944,14 @@
[:= [:field-id $id] 6]]})))))
;; if datetime string is not yyyy-MM-dd no date bucketing should take place, and thus we should get no (exact) matches
-(expect-with-non-timeseries-dbs-except #{:snowflake :bigquery}
+(qp.test/expect-with-non-timeseries-dbs-except #{:snowflake :bigquery}
;; Mongo returns empty row for count = 0. We should fix that
(case driver/*driver*
:mongo []
[[0]])
- (format-rows-by [int]
- (rows
- (data/with-temp-db [_ (checkins:1-per-day)]
+ (qp.test/format-rows-by [int]
+ (qp.test/rows
+ (data/with-db-for-dataset [_ checkins:1-per-day]
(data/run-mbql-query checkins
{:aggregation [[:count]]
:filter [:= [:field-id $timestamp] (str (du/format-date "yyyy-MM-dd" (du/date-trunc :day))
diff --git a/test/metabase/query_processor_test/remapping_test.clj b/test/metabase/query_processor_test/remapping_test.clj
index 004bfe18b4943d350842779da905d8b8810b24d4..833165faea74d2da3e10ba4a2cd291cfcbfb666e 100644
--- a/test/metabase/query_processor_test/remapping_test.clj
+++ b/test/metabase/query_processor_test/remapping_test.clj
@@ -2,7 +2,7 @@
"Tests for the remapping results"
(:require [metabase
[query-processor :as qp]
- [query-processor-test :refer :all]]
+ [query-processor-test :as qp.test :refer :all]]
[metabase.models
[dimension :refer [Dimension]]
[field :refer [Field]]]
@@ -16,10 +16,10 @@
[toucan.db :as db]))
(qp-expect-with-all-drivers
- {:rows [["20th Century Cafe" 12 "Café Sweets"]
- ["25°" 11 "Café"]
- ["33 Taps" 7 "Beer Garden"]
- ["800 Degrees Neapolitan Pizzeria" 58 "Ramen"]]
+ {:rows [["20th Century Cafe" 12 "Café"]
+ ["25°" 11 "Burger"]
+ ["33 Taps" 7 "Bar"]
+ ["800 Degrees Neapolitan Pizzeria" 58 "Pizza"]]
:columns [(data/format-name "name")
(data/format-name "category_id")
"Foo"]
@@ -33,8 +33,8 @@
{:fields [$name $category_id]
:order-by [[:asc $name]]
:limit 4})
- booleanize-native-form
- (format-rows-by [str int str])
+ qp.test/booleanize-native-form
+ (qp.test/format-rows-by [str int str])
tu/round-fingerprint-cols)))
(defn- select-columns
@@ -78,8 +78,8 @@
(->> (data/run-mbql-query venues
{:order-by [[:asc $name]]
:limit 4})
- booleanize-native-form
- (format-rows-by [int str int double double int str])
+ qp.test/booleanize-native-form
+ (qp.test/format-rows-by [int str int double double int str])
(select-columns (set (map data/format-name ["name" "price" "name_2"])))
tu/round-fingerprint-cols
data)))
@@ -107,8 +107,8 @@
{:fields [$name $price $category_id]
:order-by [[:asc $name]]
:limit 4})
- booleanize-native-form
- (format-rows-by [str int str str])
+ qp.test/booleanize-native-form
+ (qp.test/format-rows-by [str int str str])
(select-columns (set (map data/format-name ["name" "price" "name_2"])))
tu/round-fingerprint-cols
:data)))
diff --git a/test/metabase/query_processor_test/time_field_test.clj b/test/metabase/query_processor_test/time_field_test.clj
index 14f706141fde4d6cd74f9488101448cd1457ded3..10e3725e3bea1cb650ede4eabcb5564042c5a347 100644
--- a/test/metabase/query_processor_test/time_field_test.clj
+++ b/test/metabase/query_processor_test/time_field_test.clj
@@ -87,10 +87,7 @@
[[1 "Plato Yeshua" "08:30:00.000Z"]
[4 "Simcha Yan" "08:30:00.000Z"]])
(tu/with-temporary-setting-values [report-timezone "America/Los_Angeles"]
- (time-query {:filter (vec (cons
- :between
- (cons
- $last_login_time
- (if (qpt/supports-report-timezone? driver/*driver*)
- ["08:00:00" "09:00:00"]
- ["08:00:00-00:00" "09:00:00-00:00"]))))})))
+ (time-query {:filter (into [:between $last_login_time]
+ (if (qpt/supports-report-timezone? driver/*driver*)
+ ["08:00:00" "09:00:00"]
+ ["08:00:00-00:00" "09:00:00-00:00"]))})))
diff --git a/test/metabase/query_processor_test/timezones_test.clj b/test/metabase/query_processor_test/timezones_test.clj
index 7cbbad5d5498b66777a2f060795108476ef28048..6a1037db7e1536a3e970006631bbfa96a6697620 100644
--- a/test/metabase/query_processor_test/timezones_test.clj
+++ b/test/metabase/query_processor_test/timezones_test.clj
@@ -12,16 +12,10 @@
[metabase.test.data
[dataset-definitions :as defs]
[datasets :refer [expect-with-drivers]]
- [interface :as i]
[sql :as sql.tx]]
[toucan.db :as db]))
(defn- call-with-timezones-db [f]
- ;; Does the database exist?
- (when-not (i/metabase-instance defs/test-data-with-timezones driver/*driver*)
- ;; The database doesn't exist, so we need to create it
- (data/get-or-create-database! defs/test-data-with-timezones))
- ;; The database can now be used in tests
(data/with-db (data/get-or-create-database! defs/test-data-with-timezones)
(f)))
diff --git a/test/metabase/sync/sync_metadata/comments_test.clj b/test/metabase/sync/sync_metadata/comments_test.clj
index 4c788900cad18373df6c76a196a7415314bec89b..af3099044e0724c5063e5b41adc48765d5ce5cfd 100644
--- a/test/metabase/sync/sync_metadata/comments_test.clj
+++ b/test/metabase/sync/sync_metadata/comments_test.clj
@@ -21,7 +21,7 @@
(set (map (partial into {}) (db/select ['Field :name :description] :table_id [:in table-ids])))))
;; test basic field comments sync
-(tx/def-database-definition ^:const ^:private basic-field-comments
+(tx/defdataset ^:private basic-field-comments
[["basic_field_comments"
[{:field-name "with_comment", :base-type :type/Text, :field-comment "comment"}
{:field-name "no_comment", :base-type :type/Text}]
@@ -31,11 +31,11 @@
#{{:name (data/format-name "id"), :description nil}
{:name (data/format-name "with_comment"), :description "comment"}
{:name (data/format-name "no_comment"), :description nil}}
- (data/with-temp-db [db basic-field-comments]
+ (data/with-db-for-dataset [db basic-field-comments]
(db->fields db)))
;; test changing the description in metabase db so we can check it is not overwritten by comment in source db when resyncing
-(tx/def-database-definition ^:const ^:private update-desc
+(tx/defdataset ^:private update-desc
[["update_desc"
[{:field-name "updated_desc", :base-type :type/Text, :field-comment "original comment"}]
[["foo"]]]])
@@ -43,7 +43,7 @@
(datasets/expect-with-drivers #{:h2 :postgres}
#{{:name (data/format-name "id"), :description nil}
{:name (data/format-name "updated_desc"), :description "updated description"}}
- (data/with-temp-db [db update-desc]
+ (data/with-db-for-dataset [db update-desc]
;; change the description in metabase while the source table comment remains the same
(db/update-where! Field {:id (data/id "update_desc" "updated_desc")}, :description "updated description")
;; now sync the DB again, this should NOT overwrite the manually updated description
@@ -51,7 +51,7 @@
(db->fields db)))
;; test adding a comment to the source data that was initially empty, so we can check that the resync picks it up
-(tx/def-database-definition ^:const ^:private comment-after-sync
+(tx/defdataset ^:private comment-after-sync
[["comment_after_sync"
[{:field-name "comment_after_sync", :base-type :type/Text}]
[["foo"]]]])
@@ -59,14 +59,20 @@
(datasets/expect-with-drivers #{:h2 :postgres}
#{{:name (data/format-name "id"), :description nil}
{:name (data/format-name "comment_after_sync"), :description "added comment"}}
- (data/with-temp-db [db comment-after-sync]
+ (data/with-db-for-dataset [db comment-after-sync]
;; modify the source DB to add the comment and resync. The easiest way to do this is just destroy the entire DB
;; and re-create a modified version. As such, let the SQL JDBC driver know the DB is being "modified" so it can
;; destroy its current connection pool
(driver/notify-database-updated driver/*driver* db)
- (let [modified-dbdef (assoc-in comment-after-sync
- [:table-definitions 0 :field-definitions 0 :field-comment]
- "added comment")]
+ (let [modified-dbdef (update
+ comment-after-sync
+ :table-definitions
+ (fn [[tabledef]]
+ [(update
+ tabledef
+ :field-definitions
+ (fn [[fielddef]]
+ [(assoc fielddef :field-comment "added comment")]))]))]
(tx/create-db! driver/*driver* modified-dbdef))
(sync/sync-table! (Table (data/id "comment_after_sync")))
(db->fields db)))
@@ -87,13 +93,13 @@
;; test basic comments on table
(datasets/expect-with-drivers #{:h2 :postgres}
#{{:name (data/format-name "table_with_comment"), :description "table comment"}}
- (data/with-temp-db [db (basic-table "table_with_comment" "table comment")]
+ (data/with-db-for-dataset [db (basic-table "table_with_comment" "table comment")]
(db->tables db)))
;; test changing the description in metabase on table to check it is not overwritten by comment in source db when resyncing
(datasets/expect-with-drivers #{:h2 :postgres}
#{{:name (data/format-name "table_with_updated_desc"), :description "updated table description"}}
- (data/with-temp-db [db (basic-table "table_with_updated_desc" "table comment")]
+ (data/with-db-for-dataset [db (basic-table "table_with_updated_desc" "table comment")]
;; change the description in metabase while the source table comment remains the same
(db/update-where! Table {:id (data/id "table_with_updated_desc")}, :description "updated table description")
;; now sync the DB again, this should NOT overwrite the manually updated description
@@ -103,7 +109,7 @@
;; test adding a comment to the source table that was initially empty, so we can check that the resync picks it up
(datasets/expect-with-drivers #{:h2 :postgres}
#{{:name (data/format-name "table_with_comment_after_sync"), :description "added comment"}}
- (data/with-temp-db [db (basic-table "table_with_comment_after_sync" nil)]
+ (data/with-db-for-dataset [db (basic-table "table_with_comment_after_sync" nil)]
;; modify the source DB to add the comment and resync
(driver/notify-database-updated driver/*driver* db)
(tx/create-db! driver/*driver* (basic-table "table_with_comment_after_sync" "added comment"))
diff --git a/test/metabase/sync/sync_metadata/tables_test.clj b/test/metabase/sync/sync_metadata/tables_test.clj
index 696f24d5008f472741ae76924202308c9d63a089..899a4a9ceafe71c492d6bc69fd62798ec41ec7b7 100644
--- a/test/metabase/sync/sync_metadata/tables_test.clj
+++ b/test/metabase/sync/sync_metadata/tables_test.clj
@@ -11,7 +11,7 @@
[toucan.db :as db]
[toucan.util.test :as tt]))
-(tx/def-database-definition ^:private db-with-some-cruft
+(tx/defdataset ^:private db-with-some-cruft
[["acquired_toucans"
[{:field-name "species", :base-type :type/Text}
{:field-name "cam_has_acquired_one", :base-type :type/Boolean}]
diff --git a/test/metabase/test/data.clj b/test/metabase/test/data.clj
index a055b8651296771c0060d9a0a4fea222a6a500e1..2b2f210f062e68faa734f7132b85e17247d3ebe4 100644
--- a/test/metabase/test/data.clj
+++ b/test/metabase/test/data.clj
@@ -1,5 +1,38 @@
(ns metabase.test.data
- "Code related to creating and deleting test databases + datasets."
+ "Super-useful test utility functions.
+
+ Basic way stuff in here, which you'll see everywhere in the tests, is:
+
+ 1. Get the DB you're currently testing by calling `db`. Get IDs of the DB or of its Fields and Tables in that DB by
+ calling `id`.
+
+ (data/db) ; -> Get current test DB
+ (data/id) ; -> Get ID of current test DB
+ (data/id :table) ; -> Get ID of Table named `table` in current test DB
+ (data/id :table :field) ; -> Get ID of Field named `field` belonging to Table `table` in current test DB
+
+ Normally this database is the `test-data` database for the current driver, and is created the first time `db` or
+ `id` is called.
+
+ 2. Bind the current driver with `driver/with-driver`. Defaults to `:h2`
+
+ (driver/with-driver :postgres
+ (data/id))
+ ;; -> Get ID of Postgres `test-data` database, creating it if needed
+
+ 3. Bind a different database for use with for `db` and `id` functions with `with-db`.
+
+ (data/with-db [db some-database]
+ (data/id :table :field)))
+ ;; -> Return ID of Field named `field` in Table `table` in `some-db`
+
+ 4. You can use helper macros like `$ids` to replace symbols starting with `$` (for Fields) or `$$` (for Tables) with
+ calls to `id` in a form:
+
+ ($ids {:source-table $$venues, :fields [$venues.name]})
+ ;; -> {:source-table (data/id :venues), :fields [(data/id :venues :name)]}
+
+ (There are several variations of this macro; see documentation below for more details.)"
(:require [cheshire.core :as json]
[clojure
[string :as str]
@@ -22,18 +55,19 @@
[dataset-definitions :as defs]
[interface :as tx]]
[metabase.test.util.timezone :as tu.tz]
- [toucan.db :as db])
- (:import [metabase.test.data.interface DatabaseDefinition TableDefinition]))
+ [schema.core :as s]
+ [toucan.db :as db]))
(declare get-or-create-database!)
+
;;; ------------------------------------------ Dataset-Independent Data Fns ------------------------------------------
;; These functions offer a generic way to get bits of info like Table + Field IDs from any of our many driver/dataset
;; combos.
(defn get-or-create-test-data-db!
- "Get or create the Test Data database for DRIVER, which defaults to driver/`*driver*`."
+ "Get or create the Test Data database for `driver`, which defaults to `driver/*driver*`, or `:h2` if that is unbound."
([] (get-or-create-test-data-db! (tx/driver)))
([driver] (get-or-create-database! driver defs/test-data)))
@@ -54,8 +88,7 @@
(f)))
(defmacro with-db
- "Run body with DB as the current database.
- Calls to `db` and `id` use this value."
+ "Run body with `db` as the current database. Calls to `db` and `id` use this value."
[db & body]
`(do-with-db ~db (fn [] ~@body)))
@@ -98,23 +131,23 @@
to `id`.
Optionally wraps IDs in `:field-id` or `:fk->` clauses as appropriate; this defaults to true."
- [table-name body & {:keys [wrap-field-ids?], :or {wrap-field-ids? true}}]
+ [table-name-or-nil body & {:keys [wrap-field-ids?], :or {wrap-field-ids? true}}]
(walk/postwalk
(fn [form]
(cond
(not (symbol? form))
form
- (= form '$$table)
- `(id ~(keyword table-name))
+ (and table-name-or-nil (= form '$$table))
+ `(id ~(keyword table-name-or-nil))
(str/starts-with? form "$$")
- (let [table-name (str/replace form #"^\$\$" "")]
- `(id ~(keyword table-name)))
+ (let [table-name-or-nil (str/replace form #"^\$\$" "")]
+ `(id ~(keyword table-name-or-nil)))
(str/starts-with? form "$")
(let [field-name (str/replace form #"^\$" "")]
- (token->id-call wrap-field-ids? table-name field-name))
+ (token->id-call wrap-field-ids? table-name-or-nil field-name))
:else
form))
@@ -125,7 +158,7 @@
With no `.` delimiters, it is assumed we're referring to a Field belonging to `table-name`, which is passed implicitly
as the first arg. With one or more `.` delimiters, no implicit `table-name` arg is passed to `id`:
- $venue_id -> (id :sightings :venue_id) ; TABLE-NAME is implicit first arg
+ $venue_id -> (id :sightings :venue_id) ; `table-name` is implicit first arg
$cities.id -> (id :cities :id) ; specify non-default Table
Use `$$table` to refer to the table itself.
@@ -141,13 +174,16 @@
($ids [venues {:wrap-field-ids? true}]
$category_id->categories.name)
;; -> [:fk-> [:field-id (id :venues :category_id(] [:field-id (id :categories :name)]]"
- {:arglists '([table & body] [[table {:keys [wrap-field-ids?]}] & body]), :style/indent 1}
- [table-and-options & body]
- (let [[table-name options] (if (sequential? table-and-options)
- table-and-options
- [table-and-options])]
- (m/mapply $->id (keyword table-name) `(do ~@body) (merge {:wrap-field-ids? false}
- options))))
+ {:arglists '([form] [table & body] [[table {:keys [wrap-field-ids?], :or {wrap-field-ids? false}}] & body])}
+ (^{:style/indent 0} [form]
+ `($ids nil ~form))
+
+ (^{:style/indent 1} [table-and-options & body]
+ (let [[table-name options] (if (sequential? table-and-options)
+ table-and-options
+ [table-and-options])]
+ (m/mapply $->id (keyword table-name) `(do ~@body) (merge {:wrap-field-ids? false}
+ options)))))
(declare id)
@@ -266,14 +302,14 @@
(defn- add-extra-metadata!
"Add extra metadata like Field base-type, etc."
- [database-definition db]
- (doseq [^TableDefinition table-definition (:table-definitions database-definition)]
- (let [table-name (:table-name table-definition)
- table (delay (or (tx/metabase-instance table-definition db)
- (throw (Exception. (format "Table '%s' not loaded from definiton:\n%s\nFound:\n%s"
- table-name
- (u/pprint-to-str (dissoc table-definition :rows))
- (u/pprint-to-str (db/select [Table :schema :name], :db_id (:id db))))))))]
+ [{:keys [table-definitions], :as database-definition} db]
+ {:pre [(seq table-definitions)]}
+ (doseq [{:keys [table-name], :as table-definition} table-definitions]
+ (let [table (delay (or (tx/metabase-instance table-definition db)
+ (throw (Exception. (format "Table '%s' not loaded from definiton:\n%s\nFound:\n%s"
+ table-name
+ (u/pprint-to-str (dissoc table-definition :rows))
+ (u/pprint-to-str (db/select [Table :schema :name], :db_id (:id db))))))))]
(doseq [{:keys [field-name visibility-type special-type], :as field-definition} (:field-definitions table-definition)]
(let [field (delay (or (tx/metabase-instance field-definition @table)
(throw (Exception. (format "Field '%s' not loaded from definition:\n"
@@ -286,7 +322,8 @@
(log/debug (format "SET SPECIAL TYPE %s.%s -> %s" table-name field-name special-type))
(db/update! Field (:id @field) :special_type (u/keyword->qualified-name special-type))))))))
-(defn- create-database! [{:keys [database-name], :as database-definition} driver]
+(defn- create-database! [driver {:keys [database-name], :as database-definition}]
+ {:pre [(seq database-name)]}
;; Create the database and load its data
;; ALWAYS CREATE DATABASE AND LOAD DATA AS UTC! Unless you like broken tests
(tu.tz/with-jvm-tz "UTC"
@@ -299,10 +336,14 @@
;; sync newly added DB
(sync/sync-database! db)
;; add extra metadata for fields
- (add-extra-metadata! database-definition db)
+ (try
+ (add-extra-metadata! database-definition db)
+ (catch Throwable e
+ (println "Error adding extra metadata:" e)))
;; make sure we're returing an up-to-date copy of the DB
(Database (u/get-id db))))
+
(defonce ^:private ^{:arglists '([driver]), :doc "We'll have a very bad time if any sort of test runs that calls
`data/db` for the first time calls it multiple times in parallel -- for example my Oracle test that runs 30 sync
calls at the same time to make sure nothing explodes and cursors aren't leaked. To make sure this doesn't happen
@@ -318,28 +359,38 @@
(swap! locks update driver #(or % (Object.)))
(@locks driver)))))))
-(defn get-or-create-database!
- "Create DBMS database associated with DATABASE-DEFINITION, create corresponding Metabase
- `Databases`/`Tables`/`Fields`, and sync the `Database`. DRIVER should be an object that implements
- `IDriverTestExtensions`; it defaults to the value returned by the method `driver` for the current
- dataset (driver/`*driver*`), which is H2 by default."
- ([database-definition]
- (get-or-create-database! (tx/driver) database-definition))
- ([driver database-definition]
- (let [driver (tx/the-driver-with-test-extensions driver)]
+(defmulti get-or-create-database!
+ "Create DBMS database associated with `database-definition`, create corresponding Metabase Databases/Tables/Fields,
+ and sync the Database. `driver` is a keyword name of a driver that implements test extension methods (as defined in
+ the `metabase.test.data.interface` namespace); `driver` defaults to `driver/*driver*` if bound, or `:h2` if not.
+ `database-definition` is anything that implements the `tx/get-database-definition` method."
+ {:arglists '([database-definition] [driver database-definition])}
+ (fn
+ ([_]
+ (tx/dispatch-on-driver-with-test-extensions (tx/driver)))
+ ([driver _]
+ (tx/dispatch-on-driver-with-test-extensions driver)))
+ :hierarchy #'driver/hierarchy)
+
+(defmethod get-or-create-database! :default
+ ([dbdef]
+ (get-or-create-database! (tx/driver) dbdef))
+
+ ([driver dbdef]
+ (let [dbdef (tx/get-dataset-definition dbdef)]
(or
- (tx/metabase-instance database-definition driver)
+ (tx/metabase-instance dbdef driver)
(locking (driver->create-database-lock driver)
(or
- (tx/metabase-instance database-definition driver)
- (create-database! database-definition driver)))))))
+ (tx/metabase-instance dbdef driver)
+ (create-database! driver dbdef)))))))
-(defn do-with-temp-db
- "Execute F with DBDEF loaded as the current dataset. F takes a single argument, the `DatabaseInstance` that was
- loaded and synced from DBDEF."
- [^DatabaseDefinition dbdef, f]
- (let [dbdef (tx/map->DatabaseDefinition dbdef)]
+(s/defn do-with-db-for-dataset
+ "Execute `f` with `dbdef` loaded as the current dataset. `f` takes a single argument, the DatabaseInstance that was
+ loaded and synced from `dbdef`."
+ [dataset-definition, f :- (s/pred fn?)]
+ (let [dbdef (tx/get-dataset-definition dataset-definition)]
(binding [db/*disable-db-logging* true]
(let [db (get-or-create-database! (tx/driver) dbdef)]
(assert db)
@@ -348,11 +399,11 @@
(f db))))))
-(defmacro with-temp-db
- "Load and sync DATABASE-DEFINITION with DRIVER and execute BODY with the newly created `Database` bound to
- DB-BINDING, and make it the current database for `metabase.test.data` functions like `id`.
+(defmacro with-db-for-dataset
+ "Load and sync `database-definition` with `driver` and execute `body` with the newly created Database bound to
+ `db-binding`, and make it the current database for `metabase.test.data` functions like `id`.
- (with-temp-db [db tupac-sightings]
+ (with-db-for-dataset [db tupac-sightings]
(driver/process-quiery {:database (:id db)
:type :query
:query {:source-table (:id &events)
@@ -361,29 +412,29 @@
A given Database is only created once per run of the test suite, and is automatically destroyed at the conclusion
of the suite."
- [[db-binding, ^DatabaseDefinition database-definition] & body]
- `(do-with-temp-db ~database-definition
+ [[db-binding dataset-def] & body]
+ `(do-with-db-for-dataset ~dataset-def
(fn [~db-binding]
~@body)))
-(defn resolve-dbdef [symb]
- @(or (resolve symb)
+(defn resolve-dbdef [namespace-symb symb]
+ @(or (ns-resolve namespace-symb symb)
(ns-resolve 'metabase.test.data.dataset-definitions symb)
- (throw (Exception. (format "Dataset definition not found: '%s' or 'metabase.test.data.dataset-definitions/%s'"
- symb symb)))))
+ (throw (Exception. (format "Dataset definition not found: '%s/%s' or 'metabase.test.data.dataset-definitions/%s'"
+ namespace-symb symb symb)))))
(defmacro dataset
- "Load and sync a temporary `Database` defined by DATASET, make it the current DB (for `metabase.test.data` functions
- like `id`), and execute BODY.
+ "Load and sync a temporary Database defined by `dataset`, make it the current DB (for `metabase.test.data` functions
+ like `id` and `db`), and execute `body`.
- Like `with-temp-db`, but takes an unquoted symbol naming a `DatabaseDefinition` rather than the dbef itself.
- DATASET is optionally namespace-qualified; if not, `metabase.test.data.dataset-definitions` is assumed.
+ Like `with-db-for-dataset`, but takes an unquoted symbol naming a DatabaseDefinition rather than the dbef itself. `dataset`
+ is optionally namespace-qualified; if not, `metabase.test.data.dataset-definitions` is assumed.
(dataset sad-toucan-incidents
...)"
{:style/indent 1}
[dataset & body]
- `(with-temp-db [~'_ (resolve-dbdef '~dataset)]
+ `(with-db-for-dataset [~'_ (resolve-dbdef '~(ns-name *ns*) '~dataset)]
~@body))
(defn- delete-model-instance!
@@ -392,7 +443,7 @@
(db/delete! (-> instance name symbol) :id id))
(defn call-with-data
- "Takes a thunk `DATA-LOAD-FN` that returns a seq of toucan model instances that will be deleted after `BODY-FN`
+ "Takes a thunk `data-load-fn` that returns a seq of toucan model instances that will be deleted after `body-fn`
finishes"
[data-load-fn body-fn]
(let [result-instances (data-load-fn)]
@@ -402,30 +453,51 @@
(doseq [instance result-instances]
(delete-model-instance! instance))))))
-(defmacro with-data [data-load-fn & body]
+(defmacro with-data
+ "Calls `data-load-fn` to create a sequence of objects, then runs `body`; finally, deletes the objects."
+ [data-load-fn & body]
`(call-with-data ~data-load-fn (fn [] ~@body)))
-(def ^:private venue-categories
- (map vector (defs/field-values defs/test-data-map "categories" "name")))
+(defn dataset-field-values
+ "Get all the values for a field in a `dataset-definition`.
+ (dataset-field-values \"categories\" \"name\") ; -> [\"African\" \"American\" \"Artisan\" ...]"
+ ([table-name field-name]
+ (dataset-field-values defs/test-data table-name field-name))
+
+ ([dataset-definition table-name field-name]
+ (some
+ (fn [{:keys [field-definitions rows], :as tabledef}]
+ (when (= table-name (:table-name tabledef))
+ (some
+ (fn [[i fielddef]]
+ (when (= field-name (:field-name fielddef))
+ (map #(nth % i) rows)))
+ (m/indexed field-definitions))))
+ (:table-definitions (tx/get-dataset-definition dataset-definition)))))
+
+(def ^:private category-names
+ (delay (vec (dataset-field-values "categories" "name"))))
+
+;; TODO - you should always call these functions with the `with-data` macro. We should enforce this
(defn create-venue-category-remapping
- "Returns a thunk that adds an internal remapping for category_id in the venues table aliased as `REMAPPING-NAME`.
+ "Returns a thunk that adds an internal remapping for category_id in the venues table aliased as `remapping-name`.
Can be used in a `with-data` invocation."
[remapping-name]
(fn []
[(db/insert! Dimension {:field_id (id :venues :category_id)
- :name remapping-name
- :type :internal})
- (db/insert! FieldValues {:field_id (id :venues :category_id)
- :values (json/generate-string (range 0 (count venue-categories)))
- :human_readable_values (json/generate-string (map first venue-categories))})]))
+ :name remapping-name
+ :type :internal})
+ (db/insert! FieldValues {:field_id (id :venues :category_id)
+ :values (json/generate-string (range 1 (inc (count @category-names))))
+ :human_readable_values (json/generate-string @category-names)})]))
(defn create-venue-category-fk-remapping
- "Returns a thunk that adds a FK remapping for category_id in the venues table aliased as `REMAPPING-NAME`. Can be
+ "Returns a thunk that adds a FK remapping for category_id in the venues table aliased as `remapping-name`. Can be
used in a `with-data` invocation."
[remapping-name]
(fn []
- [(db/insert! Dimension {:field_id (id :venues :category_id)
- :name remapping-name
- :type :external
+ [(db/insert! Dimension {:field_id (id :venues :category_id)
+ :name remapping-name
+ :type :external
:human_readable_field_id (id :categories :name)})]))
diff --git a/test/metabase/test/data/dataset_definitions.clj b/test/metabase/test/data/dataset_definitions.clj
index 34526c3fab754cccda080c57c73d62384a5ef56a..d125ca09025e03c47cce5f0731fb75500d47fed5 100644
--- a/test/metabase/test/data/dataset_definitions.clj
+++ b/test/metabase/test/data/dataset_definitions.clj
@@ -1,45 +1,55 @@
(ns metabase.test.data.dataset-definitions
- "Definitions of various datasets for use in tests with `with-temp-db`."
- (:require [metabase.test.data.interface :as di])
+ "Definitions of various datasets for use in tests with `data/dataset` or `data/with-db-for-dataset`."
+ (:require [medley.core :as m]
+ [metabase.test.data.interface :as tx])
(:import java.sql.Time
[java.util Calendar TimeZone]))
-;; ## Datasets
+;;; +----------------------------------------------------------------------------------------------------------------+
+;;; | Various Datasets |
+;;; +----------------------------------------------------------------------------------------------------------------+
-;; The O.G. "Test Database" dataset
-(di/def-database-definition-edn test-data)
+(tx/defdataset-edn test-data
+ "The O.G. \"Test Database\" dataset.")
-;; Times when the Toucan cried
-(di/def-database-definition-edn sad-toucan-incidents)
+ (tx/defdataset-edn sad-toucan-incidents
+ "Times when the Toucan cried")
-;; Places, times, and circumstances where Tupac was sighted. Sighting timestamps are UNIX Timestamps in seconds
-(di/def-database-definition-edn tupac-sightings)
+(tx/defdataset-edn tupac-sightings
+ "Places, times, and circumstances where Tupac was sighted. Sighting timestamps are UNIX Timestamps in seconds")
-;; Dataset with nested columns, for testing a MongoDB-style database
-(di/def-database-definition-edn geographical-tips)
+(tx/defdataset-edn geographical-tips
+ "Dataset with nested columns, for testing a MongoDB-style database")
-;; A very tiny dataset with a list of places and a booleans
-(di/def-database-definition-edn places-cam-likes)
+(tx/defdataset-edn places-cam-likes
+ "A very tiny dataset with a list of places and a booleans")
-;; A small dataset with users and a set of messages between them. Each message has *2* foreign keys to user --
-;; sender and receiver -- allowing us to test situations where multiple joins for a *single* table should occur.
-(di/def-database-definition-edn avian-singles)
+(tx/defdataset-edn avian-singles
+ "A small dataset with users and a set of messages between them. Each message has *2* foreign keys to user -- sender
+ and receiver -- allowing us to test situations where multiple joins for a *single* table should occur.")
-;; A small dataset that includes an integer column with some NULL and ZERO values, meant for testing things like
-;; expressions to make sure they behave correctly
-;;
-;; As an added "bonus" this dataset has a table with a name in a slash in it, so the driver will need to support that
-;; correctly in order for this to work!
-(di/def-database-definition-edn daily-bird-counts)
+(tx/defdataset-edn daily-bird-counts
+ "A small dataset that includes an integer column with some NULL and ZERO values, meant for testing things like
+ expressions to make sure they behave correctly.
+
+ As an added bonus this dataset has a table with a name in a slash in it, so the driver will need to support that
+ correctly in order for this to work!")
+
+(tx/defdataset-edn office-checkins
+ "A small dataset that includes TIMESTAMP dates. People who stopped by the Metabase office and the time they did so.")
+
+(tx/defdataset-edn bird-flocks
+ "A small dataset with birds and the flocks they belong to (a many-to-one relationship). Some birds belong to no
+ flocks, and one flock has no birds, so this is useful for testing behavior of various types of joins. (`flock_id` is
+ not explicitly marked as a foreign key, because the test dataset syntax does not yet have a way to support nullable
+ foreign keys.)")
-;; A small dataset that includes TIMESTAMP dates. People who stopped by the Metabase office and the time they did so.
-(di/def-database-definition-edn office-checkins)
(defn- calendar-with-fields ^Calendar [date & fields]
- (let [cal-from-date (doto (Calendar/getInstance (TimeZone/getTimeZone "UTC"))
- (.setTime date))
- blank-calendar (doto (.clone cal-from-date)
- .clear)]
+ (let [^Calendar cal-from-date (doto (Calendar/getInstance (TimeZone/getTimeZone "UTC"))
+ (.setTime date))
+ ^Calendar blank-calendar (doto ^Calendar (.clone cal-from-date)
+ .clear)]
(doseq [field fields]
(.set blank-calendar field (.get cal-from-date field)))
blank-calendar))
@@ -57,66 +67,72 @@
[date]
(Time. (.getTimeInMillis (calendar-with-fields date Calendar/HOUR_OF_DAY Calendar/MINUTE Calendar/SECOND))))
-(di/def-database-definition test-data-with-time
- (di/update-table-def "users"
- (fn [table-def]
- [(first table-def)
- {:field-name "last_login_date", :base-type :type/Date}
- {:field-name "last_login_time", :base-type :type/Time}
- (peek table-def)])
- (fn [rows]
- (mapv (fn [[username last-login password-text]]
- [username (date-only last-login) (time-only last-login) password-text])
- rows))
- (for [[table-name :as orig-def] (di/slurp-edn-table-def "test-data")
- :when (= table-name "users")]
- orig-def)))
-
-(di/def-database-definition test-data-with-null-date-checkins
- (di/update-table-def "checkins"
- #(vec (concat % [{:field-name "null_only_date" :base-type :type/Date}]))
- (fn [rows]
- (mapv #(conj % nil) rows))
- (di/slurp-edn-table-def "test-data")))
-
-(di/def-database-definition test-data-with-timezones
- (di/update-table-def "users"
- (fn [table-def]
- [(first table-def)
- {:field-name "last_login", :base-type :type/DateTimeWithTZ}
- (peek table-def)])
- identity
- (di/slurp-edn-table-def "test-data")))
-
-(def test-data-map
- "Converts data from `test-data` to a map of maps like the following:
-
- {<table-name> [{<field-name> <field value> ...}]."
- (reduce (fn [acc {:keys [table-name field-definitions rows]}]
- (let [field-names (mapv :field-name field-definitions)]
- (assoc acc table-name
- (for [row rows]
- (zipmap field-names row)))))
- {} (:table-definitions test-data)))
-
-(defn field-values
- "Returns the field values for the given `TABLE` and `COLUMN` found
- in the data-map `M`."
- [m table column]
- (mapv #(get % column) (get m table)))
-
-;; Takes the `test-data` dataset and adds a `created_by` column to the users table that is self referencing
-(di/def-database-definition test-data-self-referencing-user
- (di/update-table-def "users"
- (fn [table-def]
- (conj table-def {:field-name "created_by", :base-type :type/Integer, :fk :users}))
- (fn [rows]
- (mapv (fn [[username last-login password-text] idx]
- [username last-login password-text (if (= 1 idx)
- idx
- (dec idx))])
- rows
- (iterate inc 1)))
- (for [[table-name :as orig-def] (di/slurp-edn-table-def "test-data")
- :when (= table-name "users")]
- orig-def)))
+
+(defonce ^{:doc "The main `test-data` dataset, but only the `users` table, and with `last_login_date` and
+ `last_login_time` instead of `last_login`."}
+ test-data-with-time
+ (tx/transformed-dataset-definition "test-data-with-time" test-data
+ (tx/transform-dataset-only-tables "users")
+ (tx/transform-dataset-update-table "users"
+ :table
+ (fn [tabledef]
+ (update
+ tabledef
+ :field-definitions
+ (fn [[name-field-def _ password-field-def]]
+ [name-field-def
+ (tx/map->FieldDefinition {:field-name "last_login_date", :base-type :type/Date})
+ (tx/map->FieldDefinition {:field-name "last_login_time", :base-type :type/Time})
+ password-field-def])))
+ :rows
+ (fn [rows]
+ (for [[username last-login password-text] rows]
+ [username (date-only last-login) (time-only last-login) password-text])))))
+
+(defonce ^{:doc "The main `test-data` dataset, with an additional (all-null) `null_only_date` Field."}
+ test-data-with-null-date-checkins
+ (tx/transformed-dataset-definition "test-data-with-null-date-checkins" test-data
+ (tx/transform-dataset-update-table "checkins"
+ :table
+ (fn [tabledef]
+ (update
+ tabledef
+ :field-definitions
+ concat
+ [(tx/map->FieldDefinition {:field-name "null_only_date" :base-type :type/Date})]))
+ :rows
+ (fn [rows]
+ (for [row rows]
+ (concat row [nil]))))))
+
+(defonce ^{:doc "The main `test-data` dataset, but `last_login` has a base type of `:type/DateTimeWithTZ`."}
+ test-data-with-timezones
+ (tx/transformed-dataset-definition "test-data-with-timezones" test-data
+ (tx/transform-dataset-update-table "users"
+ :table
+ (fn [tabledef]
+ (update
+ tabledef
+ :field-definitions
+ (fn [[name-field-def _ password-field-def]]
+ [name-field-def
+ (tx/map->FieldDefinition {:field-name "last_login", :base-type :type/DateTimeWithTZ})
+ password-field-def]))))))
+
+(defonce ^{:doc "The usual `test-data` dataset, but only the `users` table; adds a `created_by` column to the users
+ table that is self referencing."}
+ test-data-self-referencing-user
+ (tx/transformed-dataset-definition "test-data-self-referencing-user" test-data
+ (tx/transform-dataset-only-tables "users")
+ (tx/transform-dataset-update-table "users"
+ :table
+ (fn [tabledef]
+ (update tabledef :field-definitions concat [(tx/map->FieldDefinition
+ {:field-name "created_by", :base-type :type/Integer, :fk :users})]))
+ ;; created_by = user.id - 1, except for User 1, who was created by himself (?)
+ :rows
+ (fn [rows]
+ (for [[idx [username last-login password-text]] (m/indexed rows)]
+ [username last-login password-text (if (zero? idx)
+ 1
+ idx)])))))
diff --git a/test/metabase/test/data/dataset_definitions/bird-flocks.edn b/test/metabase/test/data/dataset_definitions/bird-flocks.edn
new file mode 100644
index 0000000000000000000000000000000000000000..7d9caeb36f616805f2a84a6be138636e486e61c8
--- /dev/null
+++ b/test/metabase/test/data/dataset_definitions/bird-flocks.edn
@@ -0,0 +1,30 @@
+[["bird" [{:field-name "name"
+ :base-type :type/Text}
+ {:field-name "flock_id"
+ :base-type :type/Integer}]
+ [["Russell Crow" 4] ; 1
+ ["Big Red" 5] ; 2
+ ["Camellia Crow" nil] ; 3
+ ["Peter Pelican" 2] ; 4
+ ["Geoff Goose" nil] ; 5
+ ["Greg Goose" 1] ; 6
+ ["Callie Crow" 4] ; 7
+ ["Patricia Pelican" nil] ; 8
+ ["Gerald Goose" 1] ; 9
+ ["Pamela Pelican" nil] ; 10
+ ["Oswald Owl" nil] ; 11
+ ["Chicken Little" 5] ; 12
+ ["Paul Pelican" 2] ; 13
+ ["McNugget" 5] ; 14
+ ["Orville Owl" 3] ; 15
+ ["Carson Crow" 4] ; 16
+ ["Olita Owl" nil] ; 17
+ ["Oliver Owl" 3]]] ; 18
+ ["flock" [{:field-name "name"
+ :base-type :type/Text}]
+ [["Green Street Gaggle"] ; 1
+ ["SoMa Squadron"] ; 2
+ ["Portrero Hill Parliament"] ; 3
+ ["Mission Street Murder"] ; 4
+ ["Bayview Brood"] ; 5
+ ["Fillmore Flock"]]]] ; 6
diff --git a/test/metabase/test/data/dataset_definitions/test-data.edn b/test/metabase/test/data/dataset_definitions/test-data.edn
index 61e70d5b22f012fb4cb37cd95433fd666bb5a637..b9da7fe8011abea58e344e092c560cab6f66d9b7 100644
--- a/test/metabase/test/data/dataset_definitions/test-data.edn
+++ b/test/metabase/test/data/dataset_definitions/test-data.edn
@@ -29,98 +29,98 @@
{:field-name "password"
:base-type :type/Text
:visibility-type :sensitive}]
- [["Plato Yeshua" #inst "2014-04-01T08:30" "4be68cda-6fd5-4ba7-944e-2b475600bda5"]
- ["Felipinho Asklepios" #inst "2014-12-05T15:15" "5bb19ad9-f3f8-421f-9750-7d398e38428d"]
- ["Kaneonuskatew Eiran" #inst "2014-11-06T16:15" "a329ccfe-b99c-42eb-9c93-cb9adc3eb1ab"]
- ["Simcha Yan" #inst "2014-01-01T08:30" "a61f97c6-4484-4a63-b37e-b5e58bfa2ecb"]
- ["Quentin Sören" #inst "2014-10-03T17:30" "10a0fea8-9bb4-48fe-a336-4d9cbbd78aa0"]
- ["Shad Ferdynand" #inst "2014-08-02T12:30" "d35c9d78-f9cf-4f52-b1cc-cb9078eebdcb"]
- ["Conchúr Tihomir" #inst "2014-08-02T09:30" "900335ad-e03b-4259-abc7-76aac21cedca"]
- ["Szymon Theutrich" #inst "2014-02-01T10:15" "d6c47a54-9d88-4c4a-8054-ace76764ed0d"]
- ["Nils Gotam" #inst "2014-04-03T09:30" "b085040c-7aa4-4e96-8c8f-420b2c99c920"]
- ["Frans Hevel" #inst "2014-07-03T19:30" "b7a43e91-9fb9-4fe9-ab6f-ea51ab0f94e4"]
- ["Spiros Teofil" #inst "2014-11-01T07:00" "62b9602c-27b8-44ea-adbd-2748f26537af"]
- ["Kfir Caj" #inst "2014-07-03T01:30" "dfe21df3-f364-479d-a5e7-04bc5d85ad2b"]
- ["Dwight Gresham" #inst "2014-08-01T10:30" "75a1ebf1-cae7-4a50-8743-32d97500f2cf"]
- ["Broen Olujimi" #inst "2014-10-03T13:45" "f9b65c74-9f91-4cfd-9248-94a53af82866"]
- ["Rüstem Hebel" #inst "2014-08-01T12:45" "02ad6b15-54b0-4491-bf0f-d781b0a2c4f5"]]]
+ [["Plato Yeshua" #inst "2014-04-01T08:30" "4be68cda-6fd5-4ba7-944e-2b475600bda5"] ; 1
+ ["Felipinho Asklepios" #inst "2014-12-05T15:15" "5bb19ad9-f3f8-421f-9750-7d398e38428d"] ; 2
+ ["Kaneonuskatew Eiran" #inst "2014-11-06T16:15" "a329ccfe-b99c-42eb-9c93-cb9adc3eb1ab"] ; 3
+ ["Simcha Yan" #inst "2014-01-01T08:30" "a61f97c6-4484-4a63-b37e-b5e58bfa2ecb"] ; 4
+ ["Quentin Sören" #inst "2014-10-03T17:30" "10a0fea8-9bb4-48fe-a336-4d9cbbd78aa0"] ; 5
+ ["Shad Ferdynand" #inst "2014-08-02T12:30" "d35c9d78-f9cf-4f52-b1cc-cb9078eebdcb"] ; 6
+ ["Conchúr Tihomir" #inst "2014-08-02T09:30" "900335ad-e03b-4259-abc7-76aac21cedca"] ; 7
+ ["Szymon Theutrich" #inst "2014-02-01T10:15" "d6c47a54-9d88-4c4a-8054-ace76764ed0d"] ; 8
+ ["Nils Gotam" #inst "2014-04-03T09:30" "b085040c-7aa4-4e96-8c8f-420b2c99c920"] ; 9
+ ["Frans Hevel" #inst "2014-07-03T19:30" "b7a43e91-9fb9-4fe9-ab6f-ea51ab0f94e4"] ; 10
+ ["Spiros Teofil" #inst "2014-11-01T07:00" "62b9602c-27b8-44ea-adbd-2748f26537af"] ; 11
+ ["Kfir Caj" #inst "2014-07-03T01:30" "dfe21df3-f364-479d-a5e7-04bc5d85ad2b"] ; 12
+ ["Dwight Gresham" #inst "2014-08-01T10:30" "75a1ebf1-cae7-4a50-8743-32d97500f2cf"] ; 13
+ ["Broen Olujimi" #inst "2014-10-03T13:45" "f9b65c74-9f91-4cfd-9248-94a53af82866"] ; 14
+ ["Rüstem Hebel" #inst "2014-08-01T12:45" "02ad6b15-54b0-4491-bf0f-d781b0a2c4f5"]]] ; 15
["categories" [{:field-name "name"
:base-type :type/Text}]
- [["African"]
- ["American"]
- ["Artisan"]
- ["Asian"]
- ["BBQ"]
- ["Bakery"]
- ["Bar"]
- ["Beer Garden"]
- ["Breakfast / Brunch"]
- ["Brewery"]
- ["Burger"]
- ["Café"]
- ["Café Sweets"]
- ["Caribbean"]
- ["Chinese"]
- ["Coffee Shop"]
- ["Comedy Club"]
- ["Deli"]
- ["Dim Sum"]
- ["Diner"]
- ["Donut Shop"]
- ["English"]
- ["Entertainment"]
- ["Fashion"]
- ["Fast Food"]
- ["Food Truck"]
- ["French"]
- ["Gay Bar"]
- ["German"]
- ["Gluten-free"]
- ["Greek"]
- ["Grocery"]
- ["Health & Beauty"]
- ["Home"]
- ["Hostel"]
- ["Hot Dog"]
- ["Hotel"]
- ["Indian"]
- ["Italian"]
- ["Japanese"]
- ["Jewish"]
- ["Juice Bar"]
- ["Karaoke"]
- ["Korean"]
- ["Landmark"]
- ["Late Dining"]
- ["Latin American"]
- ["Lounge"]
- ["Mediterannian"]
- ["Mexican"]
- ["Middle Eastern"]
- ["Molecular Gastronomy"]
- ["Moroccan"]
- ["Museum"]
- ["Nightclub"]
- ["Nightlife"]
- ["Outdoors"]
- ["Pizza"]
- ["Ramen"]
- ["Restaurant General"]
- ["Scandinavian"]
- ["Seafood"]
- ["South Pacific"]
- ["Southern"]
- ["Spanish"]
- ["Stadium"]
- ["Steakhouse"]
- ["Strip Club"]
- ["Tapas"]
- ["Tea Room"]
- ["Thai"]
- ["Unknown"]
- ["Vegetarian / Vegan"]
- ["Wine Bar"]
- ["Winery"]]]
+ [["African"] ; 1
+ ["American"] ; 2
+ ["Artisan"] ; 3
+ ["Asian"] ; 4
+ ["BBQ"] ; 5
+ ["Bakery"] ; 6
+ ["Bar"] ; 7
+ ["Beer Garden"] ; 8
+ ["Breakfast / Brunch"] ; 9
+ ["Brewery"] ; 10
+ ["Burger"] ; 11
+ ["Café"] ; 12
+ ["Café Sweets"] ; 13
+ ["Caribbean"] ; 14
+ ["Chinese"] ; 15
+ ["Coffee Shop"] ; 16
+ ["Comedy Club"] ; 17
+ ["Deli"] ; 18
+ ["Dim Sum"] ; 19
+ ["Diner"] ; 20
+ ["Donut Shop"] ; 21
+ ["English"] ; 22
+ ["Entertainment"] ; 23
+ ["Fashion"] ; 24
+ ["Fast Food"] ; 25
+ ["Food Truck"] ; 26
+ ["French"] ; 27
+ ["Gay Bar"] ; 28
+ ["German"] ; 29
+ ["Gluten-free"] ; 30
+ ["Greek"] ; 31
+ ["Grocery"] ; 32
+ ["Health & Beauty"] ; 33
+ ["Home"] ; 34
+ ["Hostel"] ; 35
+ ["Hot Dog"] ; 36
+ ["Hotel"] ; 37
+ ["Indian"] ; 38
+ ["Italian"] ; 39
+ ["Japanese"] ; 40
+ ["Jewish"] ; 41
+ ["Juice Bar"] ; 42
+ ["Karaoke"] ; 43
+ ["Korean"] ; 44
+ ["Landmark"] ; 45
+ ["Late Dining"] ; 46
+ ["Latin American"] ; 47
+ ["Lounge"] ; 48
+ ["Mediterannian"] ; 49
+ ["Mexican"] ; 50
+ ["Middle Eastern"] ; 51
+ ["Molecular Gastronomy"] ; 52
+ ["Moroccan"] ; 53
+ ["Museum"] ; 54
+ ["Nightclub"] ; 55
+ ["Nightlife"] ; 56
+ ["Outdoors"] ; 57
+ ["Pizza"] ; 58
+ ["Ramen"] ; 59
+ ["Restaurant General"] ; 60
+ ["Scandinavian"] ; 61
+ ["Seafood"] ; 62
+ ["South Pacific"] ; 63
+ ["Southern"] ; 64
+ ["Spanish"] ; 65
+ ["Stadium"] ; 66
+ ["Steakhouse"] ; 67
+ ["Strip Club"] ; 68
+ ["Tapas"] ; 69
+ ["Tea Room"] ; 70
+ ["Thai"] ; 71
+ ["Unknown"] ; 72
+ ["Vegetarian / Vegan"] ; 73
+ ["Wine Bar"] ; 74
+ ["Winery"]]] ; 75
["venues" [{:field-name "name"
:base-type :type/Text}
{:field-name "latitude"
diff --git a/test/metabase/test/data/h2.clj b/test/metabase/test/data/h2.clj
index ee62a4246e67a503269ad433ba0acfeafeef8c8a..bda97848173c907d31f7b3ea6e95858a6519b0bd 100644
--- a/test/metabase/test/data/h2.clj
+++ b/test/metabase/test/data/h2.clj
@@ -3,6 +3,8 @@
(:require [clojure.string :as str]
[metabase.db.spec :as dbspec]
[metabase.driver.sql.util :as sql.u]
+ [metabase.models.database :refer [Database]]
+ [metabase.test.data :as data]
[metabase.test.data
[interface :as tx]
[sql :as sql.tx]
@@ -10,10 +12,29 @@
[metabase.test.data.sql-jdbc
[execute :as execute]
[load-data :as load-data]
- [spec :as spec]]))
+ [spec :as spec]]
+ [toucan.db :as db]))
(sql-jdbc.tx/add-test-extensions! :h2)
+(defonce ^:private h2-test-dbs-created-by-this-instance (atom #{}))
+
+;; For H2, test databases are all in-memory, which don't work if they're saved from a different REPL session or the
+;; like. So delete any 'stale' in-mem DBs from the application DB when someone calls `get-or-create-database!` as
+;; needed.
+(defmethod data/get-or-create-database! :h2
+ ([dbdef]
+ (data/get-or-create-database! :h2 dbdef))
+
+ ([driver dbdef]
+ (let [{:keys [database-name], :as dbdef} (tx/get-dataset-definition dbdef)]
+ ;; don't think we need to bother making this super-threadsafe because REPL usage and tests are more or less
+ ;; single-threaded
+ (when (not (contains? @h2-test-dbs-created-by-this-instance database-name))
+ (db/delete! Database :engine "h2", :name database-name)
+ (swap! h2-test-dbs-created-by-this-instance conj database-name))
+ ((get-method data/get-or-create-database! :default) driver dbdef))))
+
(defmethod sql.tx/field-base-type->sql-type [:h2 :type/BigInteger] [_ _] "BIGINT")
(defmethod sql.tx/field-base-type->sql-type [:h2 :type/Boolean] [_ _] "BOOL")
(defmethod sql.tx/field-base-type->sql-type [:h2 :type/Date] [_ _] "DATE")
diff --git a/test/metabase/test/data/interface.clj b/test/metabase/test/data/interface.clj
index 2dcb7944998f7ceba76d6da06d88175be1b2e723..8082c16c2d392077c8191a432029d0b28c40ff03 100644
--- a/test/metabase/test/data/interface.clj
+++ b/test/metabase/test/data/interface.clj
@@ -8,6 +8,7 @@
(:require [clojure.string :as str]
[clojure.tools.reader.edn :as edn]
[environ.core :refer [env]]
+ [medley.core :as m]
[metabase
[db :as db]
[driver :as driver]
@@ -20,14 +21,21 @@
[metabase.test.data.env :as tx.env]
[metabase.util
[date :as du]
+ [pretty :as pretty]
[schema :as su]]
[schema.core :as s])
(:import clojure.lang.Keyword))
;;; +----------------------------------------------------------------------------------------------------------------+
-;;; | Dataset Definition Record Types |
+;;; | Dataset Definition Record Types & Protocol |
;;; +----------------------------------------------------------------------------------------------------------------+
+(defmulti get-dataset-definition
+ "Return a definition of a dataset, so a test database can be created from it."
+ {:arglists '([this])}
+ class)
+
+
(s/defrecord FieldDefinition [field-name :- su/NonBlankString
base-type :- (s/cond-pre {:native su/NonBlankString}
su/FieldType)
@@ -50,6 +58,8 @@
nil
:load-ns true)
+(defmethod get-dataset-definition DatabaseDefinition [this] this)
+
;;; +----------------------------------------------------------------------------------------------------------------+
;;; | Registering Test Extensions |
@@ -353,109 +363,208 @@
;;; | Helper Functions for Creating New Definitions |
;;; +----------------------------------------------------------------------------------------------------------------+
+(def ^:private DatasetFieldDefinition
+ "Schema for a Field in a test dataset defined by a `defdataset` form or in a dataset defnition EDN file."
+ {:field-name su/NonBlankString
+ :base-type (s/cond-pre {:native su/NonBlankString} su/FieldType)
+ (s/optional-key :special-type) (s/maybe su/FieldType)
+ (s/optional-key :visibility-type) (s/maybe (apply s/enum field/visibility-types))
+ (s/optional-key :fk) (s/maybe s/Keyword)
+ (s/optional-key :field-comment) (s/maybe su/NonBlankString)})
+
+(def ^:private DatasetTableDefinition
+ "Schema for a Table in a test dataset defined by a `defdataset` form or in a dataset defnition EDN file."
+ [(s/one su/NonBlankString "table name")
+ (s/one [DatasetFieldDefinition] "fields")
+ (s/one [[s/Any]] "rows")])
+
;; TODO - not sure everything below belongs in this namespace
-(defn create-field-definition
- "Create a new `FieldDefinition`; verify its values."
- ^FieldDefinition [field-definition-map]
+(s/defn ^:private dataset-field-definition :- FieldDefinition
+ [field-definition-map :- DatasetFieldDefinition]
+ "Parse a Field definition (from a `defdatset` form or EDN file) and return a FieldDefinition instance for
+ comsumption by various test-data-loading methods."
(s/validate FieldDefinition (map->FieldDefinition field-definition-map)))
-(defn create-table-definition
- "Convenience for creating a `TableDefinition`."
- ^TableDefinition [^String table-name, field-definition-maps rows]
- (s/validate TableDefinition (map->TableDefinition
- {:table-name table-name
- :rows rows
- :field-definitions (mapv create-field-definition field-definition-maps)})))
-
-(defn create-database-definition
- "Convenience for creating a new `DatabaseDefinition`."
+(s/defn ^:private dataset-table-definition :- TableDefinition
+ "Parse a Table definition (from a `defdatset` form or EDN file) and return a TableDefinition instance for
+ comsumption by various test-data-loading methods."
+ ([tabledef :- DatasetTableDefinition]
+ (apply dataset-table-definition tabledef))
+
+ ([table-name :- su/NonBlankString, field-definition-maps, rows]
+ (s/validate
+ TableDefinition
+ (map->TableDefinition
+ {:table-name table-name
+ :rows rows
+ :field-definitions (mapv dataset-field-definition field-definition-maps)}))))
+
+(s/defn dataset-definition :- DatabaseDefinition
+ "Parse a dataset definition (from a `defdatset` form or EDN file) and return a DatabaseDefinition instance for
+ comsumption by various test-data-loading methods."
{:style/indent 1}
- ^DatabaseDefinition [^String database-name & table-name+field-definition-maps+rows]
- (s/validate DatabaseDefinition (map->DatabaseDefinition
- {:database-name database-name
- :table-definitions (mapv (partial apply create-table-definition)
- table-name+field-definition-maps+rows)})))
+ [database-name :- su/NonBlankString, & definition]
+ (s/validate
+ DatabaseDefinition
+ (map->DatabaseDefinition
+ {:database-name database-name
+ :table-definitions (for [table definition]
+ (dataset-table-definition table))})))
+
+(defmacro defdataset
+ "Define a new dataset to test against."
+ ([dataset-name definition]
+ `(defdataset ~dataset-name nil ~definition))
+
+ ([dataset-name docstring definition]
+ {:pre [(symbol? dataset-name)]}
+ `(defonce ~(vary-meta dataset-name assoc :doc docstring, :tag `DatabaseDefinition)
+ (apply dataset-definition ~(name dataset-name) ~definition))))
+
+
+;;; +----------------------------------------------------------------------------------------------------------------+
+;;; | EDN Dataset Definitions |
+;;; +----------------------------------------------------------------------------------------------------------------+
(def ^:private edn-definitions-dir "./test/metabase/test/data/dataset_definitions/")
-(defn slurp-edn-table-def [dbname]
- ;; disabled for now when AOT compiling tests because this reads the entire file in which results in Method code too
- ;; large errors
- ;;
- ;; The fix would be to delay reading the code until runtime
- (when-not *compile-files*
- (edn/read-string (slurp (str edn-definitions-dir dbname ".edn")))))
-
-(defn update-table-def
- "Function useful for modifying a table definition before it's applied. Will invoke `UPDATE-TABLE-DEF-FN` on the vector
- of column definitions and `UPDATE-ROWS-FN` with the vector of rows in the database definition. `TABLE-DEF` is the
- database definition (typically used directly in a `def-database-definition` invocation)."
- [table-name-to-update update-table-def-fn update-rows-fn table-def]
- (vec
- (for [[table-name table-def rows :as orig-table-def] table-def]
- (if (= table-name table-name-to-update)
- [table-name
- (update-table-def-fn table-def)
- (update-rows-fn rows)]
- orig-table-def))))
-
-(defmacro def-database-definition
- "Convenience for creating a new `DatabaseDefinition` named by the symbol DATASET-NAME."
- [^clojure.lang.Symbol dataset-name table-name+field-definition-maps+rows]
- {:pre [(symbol? dataset-name)]}
- `(def ~(vary-meta dataset-name assoc :tag DatabaseDefinition)
- (apply create-database-definition ~(name dataset-name) ~table-name+field-definition-maps+rows)))
-
-(defmacro def-database-definition-edn [dbname]
- `(def-database-definition ~dbname
- ~(slurp-edn-table-def (name dbname))))
-
-;;; ## Convenience + Helper Functions
-;; TODO - should these go here, or in `metabase.test.data`?
-
-(defn get-tabledef
- "Return `TableDefinition` with TABLE-NAME in DBDEF."
- [^DatabaseDefinition dbdef, ^String table-name]
- (first (for [tabledef (:table-definitions dbdef)
- :when (= (:table-name tabledef) table-name)]
- tabledef)))
-
-(defn get-fielddefs
- "Return the `FieldDefinitions` associated with table with TABLE-NAME in DBDEF."
- [^DatabaseDefinition dbdef, ^String table-name]
- (:field-definitions (get-tabledef dbdef table-name)))
-
-(defn dbdef->table->id->k->v
+(deftype ^:private EDNDatasetDefinition [dataset-name def]
+ pretty/PrettyPrintable
+ (pretty [_]
+ (list 'edn-dataset-definition dataset-name)))
+
+(defmethod get-dataset-definition EDNDatasetDefinition
+ [^EDNDatasetDefinition this]
+ @(.def this))
+
+(s/defn edn-dataset-definition
+ "Define a new test dataset using the definition in an EDN file in the `test/metabase/test/data/dataset_definitions/`
+ directory. (Filename should be `dataset-name` + `.edn`.)"
+ [dataset-name :- su/NonBlankString]
+ (let [get-def (delay
+ (apply
+ dataset-definition
+ dataset-name
+ (edn/read-string
+ (slurp
+ (str edn-definitions-dir dataset-name ".edn")))))]
+ (EDNDatasetDefinition. dataset-name get-def)))
+
+(defmacro defdataset-edn
+ "Define a new test dataset using the definition in an EDN file in the `test/metabase/test/data/dataset_definitions/`
+ directory. (Filename should be `dataset-name` + `.edn`.)"
+ [dataset-name & [docstring]]
+ `(defonce ~(vary-meta dataset-name assoc :doc docstring, :tag `EDNDatasetDefinition)
+ (edn-dataset-definition ~(name dataset-name))))
+
+
+;;; +----------------------------------------------------------------------------------------------------------------+
+;;; | Transformed Dataset Definitions |
+;;; +----------------------------------------------------------------------------------------------------------------+
+
+(deftype ^:private TransformedDatasetDefinition [new-name wrapped-definition def]
+ pretty/PrettyPrintable
+ (pretty [_]
+ (list 'transformed-dataset-definition new-name (pretty/pretty wrapped-definition))))
+
+(s/defn transformed-dataset-definition
+ "Create a dataset definition that is a transformation of an some other one, seqentially applying `transform-fns` to
+ it. The results of `transform-fns` are cached."
+ {:style/indent 2}
+ [new-name :- su/NonBlankString, wrapped-definition & transform-fns :- [(s/pred fn?)]]
+ (let [transform-fn (apply comp (reverse transform-fns))
+ get-def (delay
+ (transform-fn
+ (assoc (get-dataset-definition wrapped-definition)
+ :database-name new-name)))]
+ (TransformedDatasetDefinition. new-name wrapped-definition get-def)))
+
+(defmethod get-dataset-definition TransformedDatasetDefinition
+ [^TransformedDatasetDefinition this]
+ @(.def this))
+
+(defn transform-dataset-update-tabledefs [f & args]
+ (fn [dbdef]
+ (apply update dbdef :table-definitions f args)))
+
+(s/defn transform-dataset-only-tables :- (s/pred fn?)
+ "Create a function for `transformed-dataset-definition` to only keep some subset of Tables from the original dataset
+ definition."
+ [& table-names]
+ (transform-dataset-update-tabledefs
+ (let [names (set table-names)]
+ (fn [tabledefs]
+ (filter
+ (fn [{:keys [table-name]}]
+ (contains? names table-name))
+ tabledefs)))))
+
+(defn transform-dataset-update-table
+ "Create a function to transform a single table, for use with `transformed-dataset-definition`. Pass `:table`, `:rows`
+ or both functions to transform the entire table definition, or just the rows, respectively."
+ {:style/indent 1}
+ [table-name & {:keys [table rows], :or {table identity, rows identity}}]
+ (transform-dataset-update-tabledefs
+ (fn [tabledefs]
+ (for [{this-name :table-name, :as tabledef} tabledefs]
+ (if (= this-name table-name)
+ (update (table tabledef) :rows rows)
+ tabledef)))))
+
+
+;;; +----------------------------------------------------------------------------------------------------------------+
+;;; | Flattening Dataset Definitions (i.e. for timeseries DBs like Druid) |
+;;; +----------------------------------------------------------------------------------------------------------------+
+
+;; TODO - maybe this should go in a different namespace
+
+(s/defn ^:private tabledef-with-name :- TableDefinition
+ "Return `TableDefinition` with `table-name` in `dbdef`."
+ [{:keys [table-definitions]} :- DatabaseDefinition, table-name :- su/NonBlankString]
+ (some
+ (fn [{this-name :table-name, :as tabledef}]
+ (when (= table-name this-name)
+ tabledef))
+ table-definitions))
+
+(s/defn ^:private fielddefs-for-table-with-name :- [FieldDefinition]
+ "Return the `FieldDefinitions` associated with table with `table-name` in `dbdef`."
+ [dbdef :- DatabaseDefinition, table-name :- su/NonBlankString]
+ (:field-definitions (tabledef-with-name dbdef table-name)))
+
+(s/defn ^:private tabledef->id->row :- {su/IntGreaterThanZero {su/NonBlankString s/Any}}
+ [{:keys [field-definitions rows]} :- TableDefinition]
+ (let [field-names (map :field-name field-definitions)]
+ (into {} (for [[i values] (m/indexed rows)]
+ [(inc i) (zipmap field-names values)]))))
+
+(s/defn ^:private dbdef->table->id->row :- {su/NonBlankString {su/IntGreaterThanZero {su/NonBlankString s/Any}}}
"Return a map of table name -> map of row ID -> map of column key -> value."
- [^DatabaseDefinition dbdef]
- (into {} (for [{:keys [table-name field-definitions rows]} (:table-definitions dbdef)]
- {table-name (let [field-names (map :field-name field-definitions)]
- (->> rows
- (map (partial zipmap field-names))
- (map-indexed (fn [i row]
- {(inc i) row}))
- (into {})))})))
-
-(defn- nest-fielddefs [^DatabaseDefinition dbdef, ^String table-name]
+ [{:keys [table-definitions]} :- DatabaseDefinition]
+ (into {} (for [{:keys [table-name] :as tabledef} table-definitions]
+ [table-name (tabledef->id->row tabledef)])))
+
+(s/defn ^:private nest-fielddefs
+ [dbdef :- DatabaseDefinition, table-name :- su/NonBlankString]
(let [nest-fielddef (fn nest-fielddef [{:keys [fk field-name], :as fielddef}]
(if-not fk
[fielddef]
(let [fk (name fk)]
- (for [nested-fielddef (mapcat nest-fielddef (get-fielddefs dbdef fk))]
+ (for [nested-fielddef (mapcat nest-fielddef (fielddefs-for-table-with-name dbdef fk))]
(update nested-fielddef :field-name (partial vector field-name fk))))))]
- (mapcat nest-fielddef (get-fielddefs dbdef table-name))))
+ (mapcat nest-fielddef (fielddefs-for-table-with-name dbdef table-name))))
-(defn- flatten-rows [^DatabaseDefinition dbdef, ^String table-name]
+(s/defn ^:private flatten-rows [dbdef :- DatabaseDefinition, table-name :- su/NonBlankString]
(let [nested-fielddefs (nest-fielddefs dbdef table-name)
- table->id->k->v (dbdef->table->id->k->v dbdef)
+ table->id->k->v (dbdef->table->id->row dbdef)
resolve-field (fn resolve-field [table id field-name]
(if (string? field-name)
(get-in table->id->k->v [table id field-name])
(let [[fk-from-name fk-table fk-dest-name] field-name
fk-id (get-in table->id->k->v [table id fk-from-name])]
(resolve-field fk-table fk-id fk-dest-name))))]
- (for [id (range 1 (inc (count (:rows (get-tabledef dbdef table-name)))))]
+ (for [id (range 1 (inc (count (:rows (tabledef-with-name dbdef table-name)))))]
(for [{:keys [field-name]} nested-fielddefs]
(resolve-field table-name id field-name)))))
@@ -468,23 +577,33 @@
(str/replace #"s$" "")
(str \_ (flatten-field-name fk-dest-name))))))
-(defn flatten-dbdef
- "Create a flattened version of DBDEF by following resolving all FKs and flattening all rows into the table with
- TABLE-NAME."
- [^DatabaseDefinition dbdef, ^String table-name]
- (create-database-definition (:database-name dbdef)
- [table-name
- (for [fielddef (nest-fielddefs dbdef table-name)]
- (update fielddef :field-name flatten-field-name))
- (flatten-rows dbdef table-name)]))
+(s/defn flattened-dataset-definition
+ "Create a flattened version of `dbdef` by following resolving all FKs and flattening all rows into the table with
+ `table-name`. For use with timeseries databases like Druid."
+ [dataset-definition, table-name :- su/NonBlankString]
+ (transformed-dataset-definition table-name dataset-definition
+ (fn [dbdef]
+ (assoc dbdef
+ :table-definitions
+ [(map->TableDefinition
+ {:table-name table-name
+ :field-definitions (for [fielddef (nest-fielddefs dbdef table-name)]
+ (update fielddef :field-name flatten-field-name))
+ :rows (flatten-rows dbdef table-name)})]))))
+
+
+;;; +----------------------------------------------------------------------------------------------------------------+
+;;; | Test Env Vars |
+;;; +----------------------------------------------------------------------------------------------------------------+
(defn db-test-env-var
- "Look up test environment var `:ENV-VAR` for the given `:DATABASE-NAME` containing connection related parameters.
+ "Look up test environment var `env-var` for the given `driver` containing connection related parameters.
If no `:default` param is specified and the var isn't found, throw.
(db-test-env-var :mysql :user) ; Look up `MB_MYSQL_TEST_USER`"
([driver env-var]
(db-test-env-var driver env-var nil))
+
([driver env-var default]
(get env
(keyword (format "mb-%s-test-%s" (name driver) (name env-var)))
diff --git a/test/metabase/timeseries_query_processor_test/util.clj b/test/metabase/timeseries_query_processor_test/util.clj
index b806d917fa31263f759ca3c6274d2b96324f9e5f..0d7f552d7c92b2c0fa1a9b69e1ac8cb6bbb38766 100644
--- a/test/metabase/timeseries_query_processor_test/util.clj
+++ b/test/metabase/timeseries_query_processor_test/util.clj
@@ -4,16 +4,14 @@
[metabase.test.data
[dataset-definitions :as defs]
[datasets :as datasets]
- [interface :as tx]]
- [metabase.util :as u]))
+ [interface :as tx]]))
(def event-based-dbs
#{:druid})
-(def flattened-db-def
- "The normal test-data DB definition as a flattened, single-table DB definition. (This is a function rather than a
- straight delay because clojure complains when they delay gets embedding in expanded macros)"
- (delay (tx/flatten-dbdef defs/test-data "checkins")))
+(def ^:private flattened-db-def
+ "The normal test-data DB definition as a flattened, single-table DB definition."
+ (tx/flattened-dataset-definition defs/test-data "checkins"))
;; force loading of the flattened db definitions for the DBs that need it
(defn- load-event-based-db-data!
@@ -21,15 +19,15 @@
[]
(doseq [driver event-based-dbs]
(datasets/with-driver-when-testing driver
- (data/do-with-temp-db @flattened-db-def (constantly nil)))))
+ (data/do-with-db-for-dataset flattened-db-def (constantly nil)))))
(defn do-with-flattened-dbdef
- "Execute F with a flattened version of the test data DB as the current DB def."
+ "Execute `f` with a flattened version of the test data DB as the current DB def."
[f]
- (data/do-with-temp-db @flattened-db-def (u/drop-first-arg f)))
+ (data/do-with-db-for-dataset flattened-db-def (fn [_] (f))))
(defmacro with-flattened-dbdef
- "Execute BODY using the flattened test data DB definition."
+ "Execute `body` using the flattened test data DB definition."
[& body]
`(do-with-flattened-dbdef (fn [] ~@body)))