Skip to content
Snippets Groups Projects
Commit 5898c9c5 authored by Cam Saül's avatar Cam Saül
Browse files

Merge pull request #2435 from metabase/druid-use-report-timezone 

Use report-timezone for Druid QP :timer:
parents 88c3d4de f5231e85
Branches
Tags
No related merge requests found
Hide whitespace changes
Inline Side-by-side
src/metabase/driver/bigquery.clj
+ 2
− 1
View file @ 5898c9c5
......@@ -182,10 +182,11 @@
"TIMESTAMP" parse-timestamp-str})
(def ^:private ^:const query-timeout-error-message "Query timed out.")
(def ^:private ^:const query-default-timeout-seconds 20)
(defn- post-process-native
([^QueryResponse response]
(post-process-native response 20))
(post-process-native response query-default-timeout-seconds))
([^QueryResponse response, ^Integer timeout-seconds]
(if-not (.getJobComplete response)
;; 99% of the time by the time this is called `.getJobComplete` will return `true`. On the off chance it doesn't, wait a few seconds for the job to finish.
......
src/metabase/driver/druid.clj
+ 17
− 23
View file @ 5898c9c5
......@@ -45,11 +45,8 @@
;;; ### Misc. Driver Fns
(defn- can-connect?
([_ details]
(can-connect? details))
([details]
(= 200 (:status (http/get (details->url details "/status"))))))
(defn- can-connect? [details]
(= 200 (:status (http/get (details->url details "/status")))))
;;; ### Query Processing
......@@ -66,20 +63,16 @@
(throw e))))
(defn- process-structured
([_ expanded-query-map]
(process-structured {:details (-> expanded-query-map :database :details)
:query (-> expanded-query-map :query)}))
([{:keys [details query]}]
(qp/process-structured-query (partial do-query details) query)))
(defn- process-structured [query]
;; Merge `:settings` into the inner query dict so the QP has access to it
(qp/process-structured-query (partial do-query (get-in query [:database :details]))
(assoc (:query query)
:settings (:settings query))))
(defn- process-native
([_ query]
(process-native query))
([{database-id :database, {query :query} :native}]
{:pre [(integer? database-id)]}
(let-404 [details (sel :one :field [Database :details] :id database-id)]
(do-query details query))))
(defn- process-native [{database-id :database, {query :query} :native}]
{:pre [(integer? database-id)]}
(let-404 [details (sel :one :field [Database :details] :id database-id)]
(do-query details query)))
;;; ### Sync
......@@ -137,7 +130,7 @@
(get-in event [:event (keyword field-name)]))]))
(defn- field-values-lazy-seq
([_ field]
([field]
(field-values-lazy-seq (:details (table/database (field/table field)))
(:name (field/table field))
(:name field)
......@@ -168,7 +161,7 @@
(u/strict-extend DruidDriver
driver/IDriver
(merge driver/IDriverDefaultsMixin
{:can-connect? can-connect?
{:can-connect? (u/drop-first-arg can-connect?)
:describe-database (u/drop-first-arg describe-database)
:describe-table (u/drop-first-arg describe-table)
:details-fields (constantly [{:name "host"
......@@ -178,8 +171,9 @@
:display-name "Broker node port"
:type :integer
:default 8082}])
:field-values-lazy-seq field-values-lazy-seq
:process-native process-native
:process-structured process-structured}))
:features (constantly #{:set-timezone})
:field-values-lazy-seq (u/drop-first-arg field-values-lazy-seq)
:process-native (u/drop-first-arg process-native)
:process-structured (u/drop-first-arg process-structured)}))
(driver/register-driver! :druid (DruidDriver.))
src/metabase/driver/druid/query_processor.clj
+ 43
− 38
View file @ 5898c9c5
(ns metabase.driver.druid.query-processor
(:require [clojure.core.match :refer [match]]
[clojure.string :as s]
[clojure.tools.logging]
[clojure.tools.logging :as log]
[metabase.query-processor :as qp]
[metabase.query-processor.interface :as i]
......@@ -26,7 +25,10 @@
(derive ::topN ::grouped-ag-query)
(derive ::groupBy ::grouped-ag-query)
(def ^:private ^:dynamic *query* nil)
(def ^:private ^:dynamic *query*
"The INNER part of the query currently being processed.
(`:settings` is merged in from the outer query as well so we can access timezone info)."
nil)
(defn- query-type-dispatch-fn [query-type & _] query-type)
......@@ -164,7 +166,8 @@
{:pre [(string? format-str)]}
{:type :timeFormat
:format format-str
:timeZone "US/Pacific" ; TODO - should we use `report-timezone` instead (?)
:timeZone (or (get-in *query* [:settings :report-timezone])
"UTC")
:locale "en-US"})
(defn- extract:js [& function-str-parts]
......@@ -172,41 +175,43 @@
{:type :javascript
:function (s/replace (apply str function-str-parts) #"\s+" " ")})
(def ^:private ^:const unit->extractionFn
(defn- unit->extractionFn
"JODA date format strings for each datetime unit. [Described here.](http://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html)."
{:default (extract:timeFormat "yyyy-MM-dd'T'HH:mm:ssZ")
:minute (extract:timeFormat "yyyy-MM-dd'T'HH:mm:00Z")
:minute-of-hour (extract:timeFormat "mm")
:hour (extract:timeFormat "yyyy-MM-dd'T'HH:00:00Z")
:hour-of-day (extract:timeFormat "HH")
:day (extract:timeFormat "yyyy-MM-ddZ")
:day-of-week (extract:js "function (timestamp) {"
" var date = new Date(timestamp);"
" return date.getDay() + 1;"
"}")
:day-of-month (extract:timeFormat "dd")
:day-of-year (extract:timeFormat "DDD")
:week (extract:js "function (timestamp) {"
" var date = new Date(timestamp);"
" var firstDOW = new Date(date - (date.getDay() * 86400000));"
" var month = firstDOW.getMonth() + 1;"
" var day = firstDOW.getDate();"
" return '' + firstDOW.getFullYear() + '-' + (month < 10 ? '0' : '') + month + '-' + (day < 10 ? '0' : '') + day;"
"}")
:week-of-year (extract:timeFormat "ww")
:month (extract:timeFormat "yyyy-MM-01")
:month-of-year (extract:timeFormat "MM")
:quarter (extract:js "function (timestamp) {"
" var date = new Date(timestamp);"
" var month = date.getMonth() + 1;" ; js months are 0 - 11
" var quarterMonth = month - ((month - 1) % 3);"
" return '' + date.getFullYear() + '-' + (quarterMonth < 10 ? '0' : '') + quarterMonth + '-01';"
"}")
:quarter-of-year (extract:js "function (timestamp) {"
" var date = new Date(timestamp);"
" return Math.floor((date.getMonth() + 3) / 3);"
"}")
:year (extract:timeFormat "yyyy")})
[unit]
(case unit
:default (extract:timeFormat "yyyy-MM-dd'T'HH:mm:ssZ")
:minute (extract:timeFormat "yyyy-MM-dd'T'HH:mm:00Z")
:minute-of-hour (extract:timeFormat "mm")
:hour (extract:timeFormat "yyyy-MM-dd'T'HH:00:00Z")
:hour-of-day (extract:timeFormat "HH")
:day (extract:timeFormat "yyyy-MM-ddZ")
:day-of-week (extract:js "function (timestamp) {"
" var date = new Date(timestamp);"
" return date.getDay() + 1;"
"}")
:day-of-month (extract:timeFormat "dd")
:day-of-year (extract:timeFormat "DDD")
:week (extract:js "function (timestamp) {"
" var date = new Date(timestamp);"
" var firstDOW = new Date(date - (date.getDay() * 86400000));"
" var month = firstDOW.getMonth() + 1;"
" var day = firstDOW.getDate();"
" return '' + firstDOW.getFullYear() + '-' + (month < 10 ? '0' : '') + month + '-' + (day < 10 ? '0' : '') + day;"
"}")
:week-of-year (extract:timeFormat "ww")
:month (extract:timeFormat "yyyy-MM-01")
:month-of-year (extract:timeFormat "MM")
:quarter (extract:js "function (timestamp) {"
" var date = new Date(timestamp);"
" var month = date.getMonth() + 1;" ; js months are 0 - 11
" var quarterMonth = month - ((month - 1) % 3);"
" return '' + date.getFullYear() + '-' + (quarterMonth < 10 ? '0' : '') + quarterMonth + '-01';"
"}")
:quarter-of-year (extract:js "function (timestamp) {"
" var date = new Date(timestamp);"
" return Math.floor((date.getMonth() + 3) / 3);"
"}")
:year (extract:timeFormat "yyyy")))
(extend-protocol IDimension
nil (->dimension-rvalue [this] (->rvalue this))
......@@ -510,7 +515,7 @@
;;; ### process-structured-query
(defn process-structured-query
"Process a structured query for a Druid DB."
"Process a structured (inner) query for a Druid DB."
[do-query query]
(binding [*query* query]
(let [[query-type druid-query] (build-druid-query query)]
......
src/metabase/models/setting.clj
+ 2
− 1
View file @ 5898c9c5
......@@ -4,7 +4,7 @@
[environ.core :as env]
[korma.core :as k]
[metabase.config :as config]
[metabase.db :refer [exists? sel del]]
[metabase.db :refer [exists? sel del], :as db]
[metabase.events :as events]
[metabase.models [common :as common]
[interface :as i]]
......@@ -248,6 +248,7 @@
(defn- restore-cache-if-needed []
(when-not @cached-setting->value
(db/setup-db-if-needed)
(reset! cached-setting->value (into {} (for [{k :key, v :value} (sel :many Setting)]
{(keyword k) v})))))
......
test/metabase/driver/timeseries_query_processor_test.clj
+ 22
− 17
View file @ 5898c9c5
......@@ -383,11 +383,11 @@
;;; date bucketing - default (day)
(expect-with-timeseries-dbs
{:columns ["timestamp" "count"]
:rows [["2013-01-03-0800" 1]
["2013-01-10-0800" 1]
["2013-01-19-0800" 1]
["2013-01-22-0800" 1]
["2013-01-23-0800" 1]]}
:rows [["2013-01-03+0000" 1]
["2013-01-10+0000" 1]
["2013-01-19+0000" 1]
["2013-01-22+0000" 1]
["2013-01-23+0000" 1]]}
(data (data/run-query checkins
(ql/aggregation (ql/count))
(ql/breakout $timestamp)
......@@ -396,7 +396,11 @@
;;; date bucketing - minute
(expect-with-timeseries-dbs
{:columns ["timestamp" "count"]
:rows [["2013-01-03T00:00:00-0800" 1] ["2013-01-10T00:00:00-0800" 1] ["2013-01-19T00:00:00-0800" 1] ["2013-01-22T00:00:00-0800" 1] ["2013-01-23T00:00:00-0800" 1]]}
:rows [["2013-01-03T08:00:00+0000" 1]
["2013-01-10T08:00:00+0000" 1]
["2013-01-19T08:00:00+0000" 1]
["2013-01-22T08:00:00+0000" 1]
["2013-01-23T08:00:00+0000" 1]]}
(data (data/run-query checkins
(ql/aggregation (ql/count))
(ql/breakout (ql/datetime-field $timestamp :minute))
......@@ -414,11 +418,11 @@
;;; date bucketing - hour
(expect-with-timeseries-dbs
{:columns ["timestamp" "count"]
:rows [["2013-01-03T00:00:00-0800" 1]
["2013-01-10T00:00:00-0800" 1]
["2013-01-19T00:00:00-0800" 1]
["2013-01-22T00:00:00-0800" 1]
["2013-01-23T00:00:00-0800" 1]]}
:rows [["2013-01-03T08:00:00+0000" 1]
["2013-01-10T08:00:00+0000" 1]
["2013-01-19T08:00:00+0000" 1]
["2013-01-22T08:00:00+0000" 1]
["2013-01-23T08:00:00+0000" 1]]}
(data (data/run-query checkins
(ql/aggregation (ql/count))
(ql/breakout (ql/datetime-field $timestamp :hour))
......@@ -427,7 +431,8 @@
;;; date bucketing - hour-of-day
(expect-with-timeseries-dbs
{:columns ["timestamp" "count"]
:rows [["00" 1000]]}
:rows [["07" 719]
["08" 281]]}
(data (data/run-query checkins
(ql/aggregation (ql/count))
(ql/breakout (ql/datetime-field $timestamp :hour-of-day))
......@@ -449,11 +454,11 @@
;;; date bucketing - day
(expect-with-timeseries-dbs
{:columns ["timestamp" "count"]
:rows [["2013-01-03-0800" 1]
["2013-01-10-0800" 1]
["2013-01-19-0800" 1]
["2013-01-22-0800" 1]
["2013-01-23-0800" 1]]}
:rows [["2013-01-03+0000" 1]
["2013-01-10+0000" 1]
["2013-01-19+0000" 1]
["2013-01-22+0000" 1]
["2013-01-23+0000" 1]]}
(data (data/run-query checkins
(ql/aggregation (ql/count))
(ql/breakout (ql/datetime-field $timestamp :day))
......
test/metabase/test/data/bigquery.clj
+ 7
− 3
View file @ 5898c9c5
......@@ -90,9 +90,13 @@
(println (u/format-color 'blue "Created BigQuery table '%s.%s'." dataset-id table-id)))
(defn- table-row-count ^Integer [^String dataset-id, ^String table-id]
(first (first (:rows (post-process-native (execute (.query (.jobs bigquery) project-id
(doto (QueryRequest.)
(.setQuery (format "SELECT COUNT(*) FROM [%s.%s]" dataset-id table-id))))))))))
(let [f (fn [] (first (first (:rows (post-process-native (execute (.query (.jobs bigquery) project-id
(doto (QueryRequest.)
(.setQuery (format "SELECT COUNT(*) FROM [%s.%s]" dataset-id table-id))))))))))]
;; automatically retry the query in case it fails / times out
(try (f)
(catch Throwable _
(f)))))
;; This is a dirty HACK
(defn- ^DateTime timestamp-korma-form->GoogleDateTime
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment