diff --git a/.circleci/config.yml b/.circleci/config.yml
index ef419db159a0169bc64931c1a642b9845b39066f..eafbecf3ddb6503681cb74dd09ee0c732fb7af5b 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -168,6 +168,18 @@ executors:
SA_PASSWORD: 'P@ssw0rd'
MSSQL_MEMORY_LIMIT_MB: 1024
+ druid:
+ working_directory: /home/circleci/metabase/metabase/
+ docker:
+ - image: metabase/ci:lein-2.9.5-clojure-1.10.3.814
+ - image: metabase/druid:0.20.2
+ environment:
+ CLUSTER_SIZE: nano-quickstart
+ # Run Docker images with 8GB or RAM instead of the default 4GB for medium instances. The Druid Docker image runs
+ # OOM all the time with the default medium size.
+ resource_class: large
+
+
fe-mongo-4:
working_directory: /home/circleci/metabase/metabase/
docker:
@@ -186,6 +198,7 @@ executors:
- image: circleci/clojure:lein-2.9.5-node-browsers
- image: metabase/qa-databases:mysql-sample-8
+
########################################################################################################################
# MAP FRAGMENTS AND CACHE KEYS #
########################################################################################################################
@@ -969,6 +982,7 @@ workflows:
name: be-tests-druid-ee
requires:
- be-tests-ee
+ e: druid
driver: druid
- test-driver:
diff --git a/modules/drivers/druid/test/metabase/driver/druid/execute_test.clj b/modules/drivers/druid/test/metabase/driver/druid/execute_test.clj
index 8ff6b419d8d9086837e4efe5a4e03d32e7eaa49f..70224ae1f28c409f2ab4af26b8917b43e12f8fb3 100644
--- a/modules/drivers/druid/test/metabase/driver/druid/execute_test.clj
+++ b/modules/drivers/druid/test/metabase/driver/druid/execute_test.clj
@@ -13,7 +13,7 @@
"columns order (#9294)")
(tqpt/with-flattened-dbdef
(let [results (mt/run-mbql-query checkins
- {:limit 1})]
+ {:limit 1, :order-by [[:asc $timestamp]]})]
(assert (= (:status results) :completed)
(u/pprint-to-str 'red results))
(testing "cols"
@@ -25,20 +25,22 @@
"venue_category_name"
"id"
"count"
+ "unique_users"
"user_name"
"user_last_login"]
(->> results :data :cols (map :name)))))
(testing "rows"
- (is (= [["2013-01-03T08:00:00Z"
+ (is (= [["2013-01-03T00:00:00Z"
"Kinaree Thai Bistro"
- "-118.344"
- "34.094"
- "1"
+ -118.344
+ 34.094
+ 1
"Thai"
- "931"
+ 931
1
+ "AQAAAQAAAAEBsA=="
"Simcha Yan"
- "2014-01-01T08:30:00.000Z"]]
+ "2014-01-01T08:30:00"]]
(-> results :data :rows)))))))))
(deftest post-process-select-query-test
diff --git a/modules/drivers/druid/test/metabase/driver/druid/query_processor_test.clj b/modules/drivers/druid/test/metabase/driver/druid/query_processor_test.clj
index 2e12be62acf44d4e89a77b9e5efeb454d060c28e..71c82eea29195c0e2fdf1dab22371e8140ab8a55 100644
--- a/modules/drivers/druid/test/metabase/driver/druid/query_processor_test.clj
+++ b/modules/drivers/druid/test/metabase/driver/druid/query_processor_test.clj
@@ -241,11 +241,11 @@
(mt/test-driver :druid
(tqpt/with-flattened-dbdef
(testing "Druid driver doesn't need to convert results to the expected timezone for us. QP middleware can handle that."
- (let [expected [["1" "The Misfit Restaurant + Bar" (t/instant "2014-04-07T07:00:00Z")]
- ["10" "Dal Rae Restaurant" (t/instant "2015-08-22T07:00:00Z")]
- ["100" "PizzaHacker" (t/instant "2014-07-26T07:00:00Z")]
- ["1000" "Tito's Tacos" (t/instant "2014-06-03T07:00:00Z")]
- ["101" "Golden Road Brewing" (t/instant "2015-09-04T07:00:00Z")]]]
+ (let [expected [[1 "The Misfit Restaurant + Bar" (t/instant "2014-04-07T00:00:00Z")]
+ [2 "Bludso's BBQ" (t/instant "2014-09-18T00:00:00Z")]
+ [3 "Philippe the Original" (t/instant "2014-09-15T00:00:00Z")]
+ [4 "Wurstküche" (t/instant "2014-03-11T00:00:00Z")]
+ [5 "Hotel Biron" (t/instant "2013-05-05T00:00:00Z")]]]
(testing "UTC timezone"
(is (= expected
(table-rows-sample))))
@@ -277,39 +277,37 @@
:database (mt/id)})
(m/dissoc-in [:data :results_metadata])))))
-(def ^:private col-defaults
- {:base_type :type/Text})
-
(deftest native-query-test
(mt/test-driver :druid
(is (= {:row_count 2
:status :completed
- :data {:rows [["931" "Simcha Yan" "1" "Kinaree Thai Bistro" 1]
- ["285" "Kfir Caj" "2" "Ruen Pair Thai Restaurant" 1]]
- :cols (mapv #(merge col-defaults %)
- [{:name "id"
- :source :native
- :display_name "id"
- :field_ref [:field "id" {:base-type :type/Text}]
- :base_type :type/Text}
- {:name "user_name"
- :source :native
- :display_name "user_name"
- :field_ref [:field "user_name" {:base-type :type/Text}]}
- {:name "venue_price"
- :source :native
- :display_name "venue_price"
- :base_type :type/Text
- :field_ref [:field "venue_price" {:base-type :type/Text}]}
- {:name "venue_name"
- :source :native
- :display_name "venue_name"
- :field_ref [:field "venue_name" {:base-type :type/Text}]}
- {:name "count"
- :source :native
- :display_name "count"
- :base_type :type/Integer
- :field_ref [:field "count" {:base-type :type/Integer}]}])
+ :data {:rows [[931 "Simcha Yan" 1 "Kinaree Thai Bistro" 1]
+ [285 "Kfir Caj" 2 "Ruen Pair Thai Restaurant" 1]]
+ :cols [{:name "id"
+ :source :native
+ :display_name "id"
+ :field_ref [:field "id" {:base-type :type/Integer}]
+ :base_type :type/Integer}
+ {:name "user_name"
+ :source :native
+ :display_name "user_name"
+ :base_type :type/Text
+ :field_ref [:field "user_name" {:base-type :type/Text}]}
+ {:name "venue_price"
+ :source :native
+ :display_name "venue_price"
+ :base_type :type/Integer
+ :field_ref [:field "venue_price" {:base-type :type/Integer}]}
+ {:name "venue_name"
+ :source :native
+ :display_name "venue_name"
+ :base_type :type/Text
+ :field_ref [:field "venue_name" {:base-type :type/Text}]}
+ {:name "count"
+ :source :native
+ :display_name "count"
+ :base_type :type/Integer
+ :field_ref [:field "count" {:base-type :type/Integer}]}]
:native_form {:query native-query-1}
:results_timezone "UTC"}}
(-> (process-native-query native-query-1)
@@ -600,7 +598,7 @@
(is (= {:filter {:type :selector, :dimension field-name, :value 1}
:queryType :scan}
(compiled query)))
- (is (= ["931" "1" "Kinaree Thai Bistro"]
+ (is (= [931 1 "Kinaree Thai Bistro"]
(mt/first-row (qp/process-query query))))))
(testing "topN query"
diff --git a/modules/drivers/druid/test/metabase/driver/druid/sync_test.clj b/modules/drivers/druid/test/metabase/driver/druid/sync_test.clj
index d460003b2e7cdedbd207b91682951567057c33c1..7f458562f67eb1ec55128cefa2fc686b72917156 100644
--- a/modules/drivers/druid/test/metabase/driver/druid/sync_test.clj
+++ b/modules/drivers/druid/test/metabase/driver/druid/sync_test.clj
@@ -14,15 +14,17 @@
(testing "describe-table"
(is (= {:schema nil
:name "checkins"
- :fields #{{:name "timestamp", :base-type :type/Instant, :database-type "timestamp", :database-position 0, :pk? false}
- {:name "venue_name", :base-type :type/Text, :database-type "STRING", :database-position 1}
- {:name "user_password", :base-type :type/Text, :database-type "STRING", :database-position 2}
- {:name "venue_longitude", :base-type :type/Text, :database-type "STRING", :database-position 3}
- {:name "venue_latitude", :base-type :type/Text, :database-type "STRING", :database-position 4}
- {:name "venue_price", :base-type :type/Text, :database-type "STRING", :database-position 5}
- {:name "venue_category_name", :base-type :type/Text, :database-type "STRING", :database-position 6}
- {:name "id", :base-type :type/Text, :database-type "STRING", :database-position 7}
- {:name "count", :base-type :type/Integer, :database-type "LONG [metric]", :database-position 8}
- {:name "user_name", :base-type :type/Text, :database-type "STRING", :database-position 9}
- {:name "user_last_login", :base-type :type/Text, :database-type "STRING", :database-position 10}}}
- (driver/describe-table :druid (mt/db) {:name "checkins"})))))))
+ :fields [{:name "timestamp", :base-type :type/Instant, :database-type "timestamp", :database-position 0, :pk? false}
+ {:name "venue_name", :base-type :type/Text, :database-type "STRING", :database-position 1}
+ {:name "user_password", :base-type :type/Text, :database-type "STRING", :database-position 2}
+ {:name "venue_longitude", :base-type :type/Float, :database-type "DOUBLE", :database-position 3}
+ {:name "venue_latitude", :base-type :type/Float, :database-type "DOUBLE", :database-position 4}
+ {:name "venue_price", :base-type :type/Integer, :database-type "LONG", :database-position 5}
+ {:name "venue_category_name", :base-type :type/Text, :database-type "STRING", :database-position 6}
+ {:name "id", :base-type :type/Integer, :database-type "LONG", :database-position 7}
+ {:name "count", :base-type :type/Integer, :database-type "LONG [metric]", :database-position 8}
+ {:name "unique_users", :base-type :type/DruidHyperUnique, :database-type "hyperUnique [metric]", :database-position 9}
+ {:name "user_name", :base-type :type/Text, :database-type "STRING", :database-position 10}
+ {:name "user_last_login", :base-type :type/Text, :database-type "STRING", :database-position 11}]}
+ (-> (driver/describe-table :druid (mt/db) {:name "checkins"})
+ (update :fields (partial sort-by :database-position)))))))))
diff --git a/modules/drivers/druid/test/metabase/test/data/druid.clj b/modules/drivers/druid/test/metabase/test/data/druid.clj
index 0c2edfc6d1d955571f670a049f7a8dd946b363a9..565a87b3f41bf1dc6af1296a2270bbba0aaa8df4 100644
--- a/modules/drivers/druid/test/metabase/test/data/druid.clj
+++ b/modules/drivers/druid/test/metabase/test/data/druid.clj
@@ -1,18 +1,50 @@
(ns metabase.test.data.druid
- (:require [metabase.test.data.impl :as tx.impl]
- [metabase.test.data.interface :as tx]))
+ (:require [clojure.string :as str]
+ [metabase.driver.druid.client :as druid.client]
+ [metabase.test.data.impl :as tx.impl]
+ [metabase.test.data.interface :as tx]
+ [metabase.util :as u]))
(tx/add-test-extensions! :druid)
+(defn- host []
+ ;; force Druid to use localhost for now -- this is until we remove the env var in CircleCI so it doesn't override
+ ;; this value.
+ (let [host "localhost" #_(tx/db-test-env-var-or-throw :druid :host "localhost")]
+ (cond->> host
+ (not (str/starts-with? host "http")) (str "http://"))))
+
+(defn- broker-port []
+ (Integer/parseUnsignedInt (tx/db-test-env-var-or-throw :druid :port "8082")))
+
+(defn- broker-port-up? []
+ (u/host-port-up? (str/replace (host) #"^https?://" "") (broker-port)))
+
(defmethod tx/dbdef->connection-details :druid
[& _]
- {:host (tx/db-test-env-var-or-throw :druid :host)
- :port (Integer/parseInt (tx/db-test-env-var-or-throw :druid :port))})
+ {:host (host)
+ :port (broker-port)})
+
+(defn- datasources-endpoint []
+ (format "%s:%d/druid/v2/datasources" (host) (broker-port)))
+
+(defn- datasources []
+ {:pre [(broker-port-up?)]}
+ (druid.client/GET (datasources-endpoint)))
+
+(defn- already-loaded []
+ (set (datasources)))
(defmethod tx/create-db! :druid
- [& _]
- nil)
+ [_ dbdef & _]
+ (let [{:keys [database-name table-definitions], :as dbdef} (tx/get-dataset-definition dbdef)]
+ (assert (= database-name "checkins")
+ "Druid tests currently only support the flattened test-data dataset.")
+ (assert (contains? (already-loaded) "checkins")
+ "Expected 'checkins' dataset to be present in Druid datasources. (This should be loaded as part of building Docker image)")
+ nil))
+;; NO-OP
(defmethod tx/destroy-db! :druid
[& _]
nil)
diff --git a/test/metabase/timeseries_query_processor_test.clj b/test/metabase/timeseries_query_processor_test.clj
index 8bf3cc56048dc97004e8d4b9abf0eb28e62ed2dc..5e43f88e9e025d2887505f904eba7bb2bd61e9ca 100644
--- a/test/metabase/timeseries_query_processor_test.clj
+++ b/test/metabase/timeseries_query_processor_test.clj
@@ -17,12 +17,13 @@
"venue_category_name"
"id"
"count"
+ "unique_users"
"user_name"
"user_last_login"]
:rows
- [["2013-01-03T08:00:00Z" "Kinaree Thai Bistro" "-118.344" "34.094" "1" "Thai" "931" 1 "Simcha Yan" "2014-01-01T08:30:00.000Z"]
- ["2013-01-10T08:00:00Z" "Ruen Pair Thai Restaurant" "-118.306" "34.1021" "2" "Thai" "285" 1 "Kfir Caj" "2014-07-03T01:30:00.000Z"]]}
+ [["2013-01-03T00:00:00Z" "Kinaree Thai Bistro" -118.344 34.094 1 "Thai" 931 1 "AQAAAQAAAAEBsA==" "Simcha Yan" "2014-01-01T08:30:00"]
+ ["2013-01-10T00:00:00Z" "Ruen Pair Thai Restaurant" -118.306 34.1021 2 "Thai" 285 1 "AQAAAQAAAAP4IA==" "Kfir Caj" "2014-07-03T01:30:00"]]}
(mt/rows+column-names
(mt/run-mbql-query checkins
{:limit 2}))))))
@@ -30,23 +31,21 @@
(deftest fields-test
(tqp.test/test-timeseries-drivers
(is (= {:columns ["venue_name" "venue_category_name" "timestamp"],
- :rows [["Kinaree Thai Bistro" "Thai" "2013-01-03T08:00:00Z"]
- ["Ruen Pair Thai Restaurant" "Thai" "2013-01-10T08:00:00Z"]]}
+ :rows [["Kinaree Thai Bistro" "Thai" "2013-01-03T00:00:00Z"]
+ ["Ruen Pair Thai Restaurant" "Thai" "2013-01-10T00:00:00Z"]]}
(mt/rows+column-names
(mt/run-mbql-query checkins
{:fields [$venue_name $venue_category_name $timestamp]
:limit 2}))))))
-;; TODO -- `:desc` tests are disabled for now, they don't seem to be working on Druid 0.11.0. Enable once we merge PR
-;; to use Druid 0.17.0
(deftest order-by-timestamp-test
(tqp.test/test-timeseries-drivers
(testing "query w/o :fields"
(doseq [[direction expected-rows]
- {#_:desc #_[["693" 1 "2015-12-29T08:00:00Z" "2014-07-03T19:30:00.000Z" "Frans Hevel" "Mexican" "34.0489" "-118.238" "Señor Fish" "2"]
- ["570" 1 "2015-12-26T08:00:00Z" "2014-07-03T01:30:00.000Z" "Kfir Caj" "Chinese" "37.7949" "-122.406" "Empress of China" "3"]]
- :asc [["2013-01-03T08:00:00Z" "Kinaree Thai Bistro" "-118.344" "34.094" "1" "Thai" "931" 1 "Simcha Yan" "2014-01-01T08:30:00.000Z"]
- ["2013-01-10T08:00:00Z" "Ruen Pair Thai Restaurant" "-118.306" "34.1021" "2" "Thai" "285" 1 "Kfir Caj" "2014-07-03T01:30:00.000Z"]]}]
+ {:desc [["2015-12-29T00:00:00Z" "Señor Fish" -118.238 34.0489 2 "Mexican" 693 1 "AQAAAQAAAAIFIA==" "Frans Hevel" "2014-07-03T19:30:00"]
+ ["2015-12-26T00:00:00Z" "Empress of China" -122.406 37.7949 3 "Chinese" 570 1 "AQAAAQAAAAP4IA==" "Kfir Caj" "2014-07-03T01:30:00"]]
+ :asc [["2013-01-03T00:00:00Z" "Kinaree Thai Bistro" -118.344 34.094 1 "Thai" 931 1 "AQAAAQAAAAEBsA==" "Simcha Yan" "2014-01-01T08:30:00"]
+ ["2013-01-10T00:00:00Z" "Ruen Pair Thai Restaurant" -118.306 34.1021 2 "Thai" 285 1 "AQAAAQAAAAP4IA==" "Kfir Caj" "2014-07-03T01:30:00"]]}]
(testing direction
(is (= {:columns ["timestamp"
"venue_name"
@@ -56,6 +55,7 @@
"venue_category_name"
"id"
"count"
+ "unique_users"
"user_name"
"user_last_login"]
:rows expected-rows}
@@ -65,12 +65,12 @@
:limit 2})))))))
(testing "for a query with :fields"
- (doseq [[direction expected-rows] {#_:desc #_[["Señor Fish" "Mexican"]
- ["Empress of China" "Chinese"]]
- :asc [["Kinaree Thai Bistro" "Thai" "2013-01-03T08:00:00Z"]
- ["Ruen Pair Thai Restaurant" "Thai" "2013-01-10T08:00:00Z"]]}]
+ (doseq [[direction expected-rows] {:desc [["Señor Fish" "Mexican" "2015-12-29T00:00:00Z"]
+ ["Empress of China" "Chinese" "2015-12-26T00:00:00Z"]]
+ :asc [["Kinaree Thai Bistro" "Thai" "2013-01-03T00:00:00Z"]
+ ["Ruen Pair Thai Restaurant" "Thai" "2013-01-10T00:00:00Z"]]}]
(testing direction
- (is (= {:columns ["venue_name" "venue_category_name" "timestamp"],
+ (is (= {:columns ["venue_name" "venue_category_name" "timestamp"]
:rows expected-rows}
(mt/rows+column-names
(mt/run-mbql-query checkins
@@ -278,11 +278,11 @@
(tqp.test/test-timeseries-drivers
(testing "filter ="
(is (= {:columns ["user_name" "venue_name" "venue_category_name" "timestamp"]
- :rows [["Plato Yeshua" "Fred 62" "Diner" "2013-03-12T07:00:00Z"]
- ["Plato Yeshua" "Dimples" "Karaoke" "2013-04-11T07:00:00Z"]
- ["Plato Yeshua" "Baby Blues BBQ" "BBQ" "2013-06-03T07:00:00Z"]
- ["Plato Yeshua" "The Daily Pint" "Bar" "2013-07-25T07:00:00Z"]
- ["Plato Yeshua" "Marlowe" "American" "2013-09-10T07:00:00Z"]]}
+ :rows [["Plato Yeshua" "Fred 62" "Diner" "2013-03-12T00:00:00Z"]
+ ["Plato Yeshua" "Dimples" "Karaoke" "2013-04-11T00:00:00Z"]
+ ["Plato Yeshua" "Baby Blues BBQ" "BBQ" "2013-06-03T00:00:00Z"]
+ ["Plato Yeshua" "The Daily Pint" "Bar" "2013-07-25T00:00:00Z"]
+ ["Plato Yeshua" "Marlowe" "American" "2013-09-10T00:00:00Z"]]}
(mt/rows+column-names
(mt/run-mbql-query checkins
{:fields [$user_name $venue_name $venue_category_name $timestamp]
@@ -300,7 +300,7 @@
(tqp.test/test-timeseries-drivers
(testing "filter AND"
(is (= {:columns ["user_name" "venue_name" "timestamp"]
- :rows [["Plato Yeshua" "The Daily Pint" "2013-07-25T07:00:00Z"]]}
+ :rows [["Plato Yeshua" "The Daily Pint" "2013-07-25T00:00:00Z"]]}
(mt/rows+column-names
(mt/run-mbql-query checkins
{:fields [$user_name $venue_name $timestamp]
@@ -489,15 +489,15 @@
(deftest minute-date-bucketing-test
(tqp.test/test-timeseries-drivers
(is (= {:columns ["timestamp" "count"]
- :rows [["2013-01-03T08:00:00+00:00" 1]
- ["2013-01-10T08:00:00+00:00" 1]
- ["2013-01-19T08:00:00+00:00" 1]
- ["2013-01-22T08:00:00+00:00" 1]
- ["2013-01-23T08:00:00+00:00" 1]]}
+ :rows [["2013-01-03T00:00:00+00:00" 1]
+ ["2013-01-10T00:00:00+00:00" 1]
+ ["2013-01-19T00:00:00+00:00" 1]
+ ["2013-01-22T00:00:00+00:00" 1]
+ ["2013-01-23T00:00:00+00:00" 1]]}
(mt/rows+column-names
(mt/run-mbql-query checkins
{:aggregation [[:count]]
- :breakout [[:datetime-field $timestamp :minute]]
+ :breakout [[:field %timestamp {:temporal-unit :minute}]]
:limit 5}))))))
(deftest date-bucketing-test
@@ -505,13 +505,12 @@
(tqp.test/with-flattened-dbdef
(doseq [[unit expected-rows]
{:minute-of-hour [[0 1000]]
- :hour [["2013-01-03T08:00:00+00:00" 1]
- ["2013-01-10T08:00:00+00:00" 1]
- ["2013-01-19T08:00:00+00:00" 1]
- ["2013-01-22T08:00:00+00:00" 1]
- ["2013-01-23T08:00:00+00:00" 1]]
- :hour-of-day [[7 719]
- [8 281]]
+ :hour [["2013-01-03T00:00:00+00:00" 1]
+ ["2013-01-10T00:00:00+00:00" 1]
+ ["2013-01-19T00:00:00+00:00" 1]
+ ["2013-01-22T00:00:00+00:00" 1]
+ ["2013-01-23T00:00:00+00:00" 1]]
+ :hour-of-day [[0 1000]]
:week [["2012-12-30" 1]
["2013-01-06" 1]
["2013-01-13" 1]
@@ -569,7 +568,7 @@
(let [{:keys [columns rows]} (mt/rows+column-names
(mt/run-mbql-query checkins
{:aggregation [[:count]]
- :breakout [[:datetime-field $timestamp unit]]
+ :breakout [[:field %timestamp {:temporal-unit unit}]]
:limit 5}))]
(is (= ["timestamp" "count"]
columns))
@@ -581,7 +580,7 @@
(let [{:keys [columns rows]} (mt/rows+column-names
(mt/run-mbql-query checkins
{:aggregation [[:count]]
- :breakout [[:datetime-field $timestamp unit]]}))]
+ :breakout [[:field %timestamp {:temporal-unit unit}]]}))]
(is (= ["timestamp" "count"]
columns))
(is (= expected-rows