Skip to content
Snippets Groups Projects
Unverified Commit afd5eafa authored by Cam Saul's avatar Cam Saul Committed by GitHub
Browse files

Druid docker image (#15873) 

* Druid Docker image [WIP]

* More Druid Docker stuff [WIP]

* Test fixes :wrench:

* Remove maxRowsQueuedForOrdering from query, don't think we need it

* Force test Druid host -> localhost for now

* Update sync tests for updated test data

* Revert unused changes
parent a815de18
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Showing
with 143 additions and 96 deletions
.circleci/config.yml
+ 14
− 0
View file @ afd5eafa
......@@ -168,6 +168,18 @@ executors:
SA_PASSWORD: 'P@ssw0rd'
MSSQL_MEMORY_LIMIT_MB: 1024
druid:
working_directory: /home/circleci/metabase/metabase/
docker:
- image: metabase/ci:lein-2.9.5-clojure-1.10.3.814
- image: metabase/druid:0.20.2
environment:
CLUSTER_SIZE: nano-quickstart
# Run Docker images with 8GB or RAM instead of the default 4GB for medium instances. The Druid Docker image runs
# OOM all the time with the default medium size.
resource_class: large
fe-mongo-4:
working_directory: /home/circleci/metabase/metabase/
docker:
......@@ -186,6 +198,7 @@ executors:
- image: circleci/clojure:lein-2.9.5-node-browsers
- image: metabase/qa-databases:mysql-sample-8
########################################################################################################################
# MAP FRAGMENTS AND CACHE KEYS #
########################################################################################################################
......@@ -969,6 +982,7 @@ workflows:
name: be-tests-druid-ee
requires:
- be-tests-ee
e: druid
driver: druid
- test-driver:
......
modules/drivers/druid/test/metabase/driver/druid/execute_test.clj
+ 9
− 7
View file @ afd5eafa
......@@ -13,7 +13,7 @@
"columns order (#9294)")
(tqpt/with-flattened-dbdef
(let [results (mt/run-mbql-query checkins
{:limit 1})]
{:limit 1, :order-by [[:asc $timestamp]]})]
(assert (= (:status results) :completed)
(u/pprint-to-str 'red results))
(testing "cols"
......@@ -25,20 +25,22 @@
"venue_category_name"
"id"
"count"
"unique_users"
"user_name"
"user_last_login"]
(->> results :data :cols (map :name)))))
(testing "rows"
(is (= [["2013-01-03T08:00:00Z"
(is (= [["2013-01-03T00:00:00Z"
"Kinaree Thai Bistro"
"-118.344"
"34.094"
"1"
-118.344
34.094
1
"Thai"
"931"
931
1
"AQAAAQAAAAEBsA=="
"Simcha Yan"
"2014-01-01T08:30:00.000Z"]]
"2014-01-01T08:30:00"]]
(-> results :data :rows)))))))))
(deftest post-process-select-query-test
......
modules/drivers/druid/test/metabase/driver/druid/query_processor_test.clj
+ 33
− 35
View file @ afd5eafa
......@@ -241,11 +241,11 @@
(mt/test-driver :druid
(tqpt/with-flattened-dbdef
(testing "Druid driver doesn't need to convert results to the expected timezone for us. QP middleware can handle that."
(let [expected [["1" "The Misfit Restaurant + Bar" (t/instant "2014-04-07T07:00:00Z")]
["10" "Dal Rae Restaurant" (t/instant "2015-08-22T07:00:00Z")]
["100" "PizzaHacker" (t/instant "2014-07-26T07:00:00Z")]
["1000" "Tito's Tacos" (t/instant "2014-06-03T07:00:00Z")]
["101" "Golden Road Brewing" (t/instant "2015-09-04T07:00:00Z")]]]
(let [expected [[1 "The Misfit Restaurant + Bar" (t/instant "2014-04-07T00:00:00Z")]
[2 "Bludso's BBQ" (t/instant "2014-09-18T00:00:00Z")]
[3 "Philippe the Original" (t/instant "2014-09-15T00:00:00Z")]
[4 "Wurstküche" (t/instant "2014-03-11T00:00:00Z")]
[5 "Hotel Biron" (t/instant "2013-05-05T00:00:00Z")]]]
(testing "UTC timezone"
(is (= expected
(table-rows-sample))))
......@@ -277,39 +277,37 @@
:database (mt/id)})
(m/dissoc-in [:data :results_metadata])))))
(def ^:private col-defaults
{:base_type :type/Text})
(deftest native-query-test
(mt/test-driver :druid
(is (= {:row_count 2
:status :completed
:data {:rows [["931" "Simcha Yan" "1" "Kinaree Thai Bistro" 1]
["285" "Kfir Caj" "2" "Ruen Pair Thai Restaurant" 1]]
:cols (mapv #(merge col-defaults %)
[{:name "id"
:source :native
:display_name "id"
:field_ref [:field "id" {:base-type :type/Text}]
:base_type :type/Text}
{:name "user_name"
:source :native
:display_name "user_name"
:field_ref [:field "user_name" {:base-type :type/Text}]}
{:name "venue_price"
:source :native
:display_name "venue_price"
:base_type :type/Text
:field_ref [:field "venue_price" {:base-type :type/Text}]}
{:name "venue_name"
:source :native
:display_name "venue_name"
:field_ref [:field "venue_name" {:base-type :type/Text}]}
{:name "count"
:source :native
:display_name "count"
:base_type :type/Integer
:field_ref [:field "count" {:base-type :type/Integer}]}])
:data {:rows [[931 "Simcha Yan" 1 "Kinaree Thai Bistro" 1]
[285 "Kfir Caj" 2 "Ruen Pair Thai Restaurant" 1]]
:cols [{:name "id"
:source :native
:display_name "id"
:field_ref [:field "id" {:base-type :type/Integer}]
:base_type :type/Integer}
{:name "user_name"
:source :native
:display_name "user_name"
:base_type :type/Text
:field_ref [:field "user_name" {:base-type :type/Text}]}
{:name "venue_price"
:source :native
:display_name "venue_price"
:base_type :type/Integer
:field_ref [:field "venue_price" {:base-type :type/Integer}]}
{:name "venue_name"
:source :native
:display_name "venue_name"
:base_type :type/Text
:field_ref [:field "venue_name" {:base-type :type/Text}]}
{:name "count"
:source :native
:display_name "count"
:base_type :type/Integer
:field_ref [:field "count" {:base-type :type/Integer}]}]
:native_form {:query native-query-1}
:results_timezone "UTC"}}
(-> (process-native-query native-query-1)
......@@ -600,7 +598,7 @@
(is (= {:filter {:type :selector, :dimension field-name, :value 1}
:queryType :scan}
(compiled query)))
(is (= ["931" "1" "Kinaree Thai Bistro"]
(is (= [931 1 "Kinaree Thai Bistro"]
(mt/first-row (qp/process-query query))))))
(testing "topN query"
......
modules/drivers/druid/test/metabase/driver/druid/sync_test.clj
+ 14
− 12
View file @ afd5eafa
......@@ -14,15 +14,17 @@
(testing "describe-table"
(is (= {:schema nil
:name "checkins"
:fields #{{:name "timestamp", :base-type :type/Instant, :database-type "timestamp", :database-position 0, :pk? false}
{:name "venue_name", :base-type :type/Text, :database-type "STRING", :database-position 1}
{:name "user_password", :base-type :type/Text, :database-type "STRING", :database-position 2}
{:name "venue_longitude", :base-type :type/Text, :database-type "STRING", :database-position 3}
{:name "venue_latitude", :base-type :type/Text, :database-type "STRING", :database-position 4}
{:name "venue_price", :base-type :type/Text, :database-type "STRING", :database-position 5}
{:name "venue_category_name", :base-type :type/Text, :database-type "STRING", :database-position 6}
{:name "id", :base-type :type/Text, :database-type "STRING", :database-position 7}
{:name "count", :base-type :type/Integer, :database-type "LONG [metric]", :database-position 8}
{:name "user_name", :base-type :type/Text, :database-type "STRING", :database-position 9}
{:name "user_last_login", :base-type :type/Text, :database-type "STRING", :database-position 10}}}
(driver/describe-table :druid (mt/db) {:name "checkins"})))))))
:fields [{:name "timestamp", :base-type :type/Instant, :database-type "timestamp", :database-position 0, :pk? false}
{:name "venue_name", :base-type :type/Text, :database-type "STRING", :database-position 1}
{:name "user_password", :base-type :type/Text, :database-type "STRING", :database-position 2}
{:name "venue_longitude", :base-type :type/Float, :database-type "DOUBLE", :database-position 3}
{:name "venue_latitude", :base-type :type/Float, :database-type "DOUBLE", :database-position 4}
{:name "venue_price", :base-type :type/Integer, :database-type "LONG", :database-position 5}
{:name "venue_category_name", :base-type :type/Text, :database-type "STRING", :database-position 6}
{:name "id", :base-type :type/Integer, :database-type "LONG", :database-position 7}
{:name "count", :base-type :type/Integer, :database-type "LONG [metric]", :database-position 8}
{:name "unique_users", :base-type :type/DruidHyperUnique, :database-type "hyperUnique [metric]", :database-position 9}
{:name "user_name", :base-type :type/Text, :database-type "STRING", :database-position 10}
{:name "user_last_login", :base-type :type/Text, :database-type "STRING", :database-position 11}]}
(-> (driver/describe-table :druid (mt/db) {:name "checkins"})
(update :fields (partial sort-by :database-position)))))))))
modules/drivers/druid/test/metabase/test/data/druid.clj
+ 38
− 6
View file @ afd5eafa
(ns metabase.test.data.druid
(:require [metabase.test.data.impl :as tx.impl]
[metabase.test.data.interface :as tx]))
(:require [clojure.string :as str]
[metabase.driver.druid.client :as druid.client]
[metabase.test.data.impl :as tx.impl]
[metabase.test.data.interface :as tx]
[metabase.util :as u]))
(tx/add-test-extensions! :druid)
(defn- host []
;; force Druid to use localhost for now -- this is until we remove the env var in CircleCI so it doesn't override
;; this value.
(let [host "localhost" #_(tx/db-test-env-var-or-throw :druid :host "localhost")]
(cond->> host
(not (str/starts-with? host "http")) (str "http://"))))
(defn- broker-port []
(Integer/parseUnsignedInt (tx/db-test-env-var-or-throw :druid :port "8082")))
(defn- broker-port-up? []
(u/host-port-up? (str/replace (host) #"^https?://" "") (broker-port)))
(defmethod tx/dbdef->connection-details :druid
[& _]
{:host (tx/db-test-env-var-or-throw :druid :host)
:port (Integer/parseInt (tx/db-test-env-var-or-throw :druid :port))})
{:host (host)
:port (broker-port)})
(defn- datasources-endpoint []
(format "%s:%d/druid/v2/datasources" (host) (broker-port)))
(defn- datasources []
{:pre [(broker-port-up?)]}
(druid.client/GET (datasources-endpoint)))
(defn- already-loaded []
(set (datasources)))
(defmethod tx/create-db! :druid
[& _]
nil)
[_ dbdef & _]
(let [{:keys [database-name table-definitions], :as dbdef} (tx/get-dataset-definition dbdef)]
(assert (= database-name "checkins")
"Druid tests currently only support the flattened test-data dataset.")
(assert (contains? (already-loaded) "checkins")
"Expected 'checkins' dataset to be present in Druid datasources. (This should be loaded as part of building Docker image)")
nil))
;; NO-OP
(defmethod tx/destroy-db! :druid
[& _]
nil)
......
test/metabase/timeseries_query_processor_test.clj
+ 35
− 36
View file @ afd5eafa
......@@ -17,12 +17,13 @@
"venue_category_name"
"id"
"count"
"unique_users"
"user_name"
"user_last_login"]
:rows
[["2013-01-03T08:00:00Z" "Kinaree Thai Bistro" "-118.344" "34.094" "1" "Thai" "931" 1 "Simcha Yan" "2014-01-01T08:30:00.000Z"]
["2013-01-10T08:00:00Z" "Ruen Pair Thai Restaurant" "-118.306" "34.1021" "2" "Thai" "285" 1 "Kfir Caj" "2014-07-03T01:30:00.000Z"]]}
[["2013-01-03T00:00:00Z" "Kinaree Thai Bistro" -118.344 34.094 1 "Thai" 931 1 "AQAAAQAAAAEBsA==" "Simcha Yan" "2014-01-01T08:30:00"]
["2013-01-10T00:00:00Z" "Ruen Pair Thai Restaurant" -118.306 34.1021 2 "Thai" 285 1 "AQAAAQAAAAP4IA==" "Kfir Caj" "2014-07-03T01:30:00"]]}
(mt/rows+column-names
(mt/run-mbql-query checkins
{:limit 2}))))))
......@@ -30,23 +31,21 @@
(deftest fields-test
(tqp.test/test-timeseries-drivers
(is (= {:columns ["venue_name" "venue_category_name" "timestamp"],
:rows [["Kinaree Thai Bistro" "Thai" "2013-01-03T08:00:00Z"]
["Ruen Pair Thai Restaurant" "Thai" "2013-01-10T08:00:00Z"]]}
:rows [["Kinaree Thai Bistro" "Thai" "2013-01-03T00:00:00Z"]
["Ruen Pair Thai Restaurant" "Thai" "2013-01-10T00:00:00Z"]]}
(mt/rows+column-names
(mt/run-mbql-query checkins
{:fields [$venue_name $venue_category_name $timestamp]
:limit 2}))))))
;; TODO -- `:desc` tests are disabled for now, they don't seem to be working on Druid 0.11.0. Enable once we merge PR
;; to use Druid 0.17.0
(deftest order-by-timestamp-test
(tqp.test/test-timeseries-drivers
(testing "query w/o :fields"
(doseq [[direction expected-rows]
{#_:desc #_[["693" 1 "2015-12-29T08:00:00Z" "2014-07-03T19:30:00.000Z" "Frans Hevel" "Mexican" "34.0489" "-118.238" "Señor Fish" "2"]
["570" 1 "2015-12-26T08:00:00Z" "2014-07-03T01:30:00.000Z" "Kfir Caj" "Chinese" "37.7949" "-122.406" "Empress of China" "3"]]
:asc [["2013-01-03T08:00:00Z" "Kinaree Thai Bistro" "-118.344" "34.094" "1" "Thai" "931" 1 "Simcha Yan" "2014-01-01T08:30:00.000Z"]
["2013-01-10T08:00:00Z" "Ruen Pair Thai Restaurant" "-118.306" "34.1021" "2" "Thai" "285" 1 "Kfir Caj" "2014-07-03T01:30:00.000Z"]]}]
{:desc [["2015-12-29T00:00:00Z" "Señor Fish" -118.238 34.0489 2 "Mexican" 693 1 "AQAAAQAAAAIFIA==" "Frans Hevel" "2014-07-03T19:30:00"]
["2015-12-26T00:00:00Z" "Empress of China" -122.406 37.7949 3 "Chinese" 570 1 "AQAAAQAAAAP4IA==" "Kfir Caj" "2014-07-03T01:30:00"]]
:asc [["2013-01-03T00:00:00Z" "Kinaree Thai Bistro" -118.344 34.094 1 "Thai" 931 1 "AQAAAQAAAAEBsA==" "Simcha Yan" "2014-01-01T08:30:00"]
["2013-01-10T00:00:00Z" "Ruen Pair Thai Restaurant" -118.306 34.1021 2 "Thai" 285 1 "AQAAAQAAAAP4IA==" "Kfir Caj" "2014-07-03T01:30:00"]]}]
(testing direction
(is (= {:columns ["timestamp"
"venue_name"
......@@ -56,6 +55,7 @@
"venue_category_name"
"id"
"count"
"unique_users"
"user_name"
"user_last_login"]
:rows expected-rows}
......@@ -65,12 +65,12 @@
:limit 2})))))))
(testing "for a query with :fields"
(doseq [[direction expected-rows] {#_:desc #_[["Señor Fish" "Mexican"]
["Empress of China" "Chinese"]]
:asc [["Kinaree Thai Bistro" "Thai" "2013-01-03T08:00:00Z"]
["Ruen Pair Thai Restaurant" "Thai" "2013-01-10T08:00:00Z"]]}]
(doseq [[direction expected-rows] {:desc [["Señor Fish" "Mexican" "2015-12-29T00:00:00Z"]
["Empress of China" "Chinese" "2015-12-26T00:00:00Z"]]
:asc [["Kinaree Thai Bistro" "Thai" "2013-01-03T00:00:00Z"]
["Ruen Pair Thai Restaurant" "Thai" "2013-01-10T00:00:00Z"]]}]
(testing direction
(is (= {:columns ["venue_name" "venue_category_name" "timestamp"],
(is (= {:columns ["venue_name" "venue_category_name" "timestamp"]
:rows expected-rows}
(mt/rows+column-names
(mt/run-mbql-query checkins
......@@ -278,11 +278,11 @@
(tqp.test/test-timeseries-drivers
(testing "filter ="
(is (= {:columns ["user_name" "venue_name" "venue_category_name" "timestamp"]
:rows [["Plato Yeshua" "Fred 62" "Diner" "2013-03-12T07:00:00Z"]
["Plato Yeshua" "Dimples" "Karaoke" "2013-04-11T07:00:00Z"]
["Plato Yeshua" "Baby Blues BBQ" "BBQ" "2013-06-03T07:00:00Z"]
["Plato Yeshua" "The Daily Pint" "Bar" "2013-07-25T07:00:00Z"]
["Plato Yeshua" "Marlowe" "American" "2013-09-10T07:00:00Z"]]}
:rows [["Plato Yeshua" "Fred 62" "Diner" "2013-03-12T00:00:00Z"]
["Plato Yeshua" "Dimples" "Karaoke" "2013-04-11T00:00:00Z"]
["Plato Yeshua" "Baby Blues BBQ" "BBQ" "2013-06-03T00:00:00Z"]
["Plato Yeshua" "The Daily Pint" "Bar" "2013-07-25T00:00:00Z"]
["Plato Yeshua" "Marlowe" "American" "2013-09-10T00:00:00Z"]]}
(mt/rows+column-names
(mt/run-mbql-query checkins
{:fields [$user_name $venue_name $venue_category_name $timestamp]
......@@ -300,7 +300,7 @@
(tqp.test/test-timeseries-drivers
(testing "filter AND"
(is (= {:columns ["user_name" "venue_name" "timestamp"]
:rows [["Plato Yeshua" "The Daily Pint" "2013-07-25T07:00:00Z"]]}
:rows [["Plato Yeshua" "The Daily Pint" "2013-07-25T00:00:00Z"]]}
(mt/rows+column-names
(mt/run-mbql-query checkins
{:fields [$user_name $venue_name $timestamp]
......@@ -489,15 +489,15 @@
(deftest minute-date-bucketing-test
(tqp.test/test-timeseries-drivers
(is (= {:columns ["timestamp" "count"]
:rows [["2013-01-03T08:00:00+00:00" 1]
["2013-01-10T08:00:00+00:00" 1]
["2013-01-19T08:00:00+00:00" 1]
["2013-01-22T08:00:00+00:00" 1]
["2013-01-23T08:00:00+00:00" 1]]}
:rows [["2013-01-03T00:00:00+00:00" 1]
["2013-01-10T00:00:00+00:00" 1]
["2013-01-19T00:00:00+00:00" 1]
["2013-01-22T00:00:00+00:00" 1]
["2013-01-23T00:00:00+00:00" 1]]}
(mt/rows+column-names
(mt/run-mbql-query checkins
{:aggregation [[:count]]
:breakout [[:datetime-field $timestamp :minute]]
:breakout [[:field %timestamp {:temporal-unit :minute}]]
:limit 5}))))))
(deftest date-bucketing-test
......@@ -505,13 +505,12 @@
(tqp.test/with-flattened-dbdef
(doseq [[unit expected-rows]
{:minute-of-hour [[0 1000]]
:hour [["2013-01-03T08:00:00+00:00" 1]
["2013-01-10T08:00:00+00:00" 1]
["2013-01-19T08:00:00+00:00" 1]
["2013-01-22T08:00:00+00:00" 1]
["2013-01-23T08:00:00+00:00" 1]]
:hour-of-day [[7 719]
[8 281]]
:hour [["2013-01-03T00:00:00+00:00" 1]
["2013-01-10T00:00:00+00:00" 1]
["2013-01-19T00:00:00+00:00" 1]
["2013-01-22T00:00:00+00:00" 1]
["2013-01-23T00:00:00+00:00" 1]]
:hour-of-day [[0 1000]]
:week [["2012-12-30" 1]
["2013-01-06" 1]
["2013-01-13" 1]
......@@ -569,7 +568,7 @@
(let [{:keys [columns rows]} (mt/rows+column-names
(mt/run-mbql-query checkins
{:aggregation [[:count]]
:breakout [[:datetime-field $timestamp unit]]
:breakout [[:field %timestamp {:temporal-unit unit}]]
:limit 5}))]
(is (= ["timestamp" "count"]
columns))
......@@ -581,7 +580,7 @@
(let [{:keys [columns rows]} (mt/rows+column-names
(mt/run-mbql-query checkins
{:aggregation [[:count]]
:breakout [[:datetime-field $timestamp unit]]}))]
:breakout [[:field %timestamp {:temporal-unit unit}]]}))]
(is (= ["timestamp" "count"]
columns))
(is (= expected-rows
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment