Skip to content
Snippets Groups Projects
Commit 5dba8471 authored by Cam Saül's avatar Cam Saül Committed by GitHub
Browse files

Merge pull request #3459 from metabase/swisscom-bigdata-druid-timezone-filter 

Druid filters in configured timezone (credit: @moumny)
parents 30baf9fa 1f8c5cac
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
src/metabase/driver/druid/query_processor.clj
+ 5
− 4
View file @ 5dba8471
......@@ -36,6 +36,8 @@
(`:settings` is merged in from the outer query as well so we can access timezone info)."
nil)
(defn- get-timezone-id [] (or (get-in *query* [:settings :report-timezone]) "UTC"))
(defn- query-type-dispatch-fn [query-type & _] query-type)
(defprotocol ^:private IRValue
......@@ -51,8 +53,8 @@
Field (->rvalue [this] (:field-name this))
DateTimeField (->rvalue [this] (->rvalue (:field this)))
Value (->rvalue [this] (:value this))
DateTimeValue (->rvalue [{{unit :unit} :field, value :value}] (u/date->iso-8601 (u/date-trunc-or-extract unit value)))
RelativeDateTimeValue (->rvalue [{:keys [unit amount]}] (u/date->iso-8601 (u/date-trunc-or-extract unit (u/relative-date unit amount)))))
DateTimeValue (->rvalue [{{unit :unit} :field, value :value}] (u/date->iso-8601 (u/date-trunc-or-extract unit value (get-timezone-id))))
RelativeDateTimeValue (->rvalue [{:keys [unit amount]}] (u/date->iso-8601 (u/date-trunc-or-extract unit (u/relative-date unit amount) (get-timezone-id)))))
(defprotocol ^:private IDimensionOrMetric
(^:private dimension-or-metric? [this]
......@@ -238,8 +240,7 @@
:month "P1M"
:quarter "P3M"
:year "P1Y")
:timeZone (or (get-in *query* [:settings :report-timezone])
"UTC")})
:timeZone (get-timezone-id)})
(def ^:private ^:const units-that-need-post-processing-int-parsing
"`extract:timeFormat` always returns a string; there are cases where we'd like to return an integer instead, such as `:day-of-month`.
......
src/metabase/util.clj
+ 49
− 25
View file @ 5dba8471
......@@ -127,6 +127,7 @@
(try (->Timestamp s)
(catch Throwable e)))))
(defn ->Date
"Coerece DATE to a `java.util.Date`."
(^java.util.Date []
......@@ -134,6 +135,7 @@
(^java.util.Date [date]
(java.util.Date. (.getTime (->Timestamp date)))))
(defn ->Calendar
"Coerce DATE to a `java.util.Calendar`."
(^java.util.Calendar []
......@@ -141,7 +143,11 @@
(.setTimeZone (TimeZone/getTimeZone "UTC"))))
(^java.util.Calendar [date]
(doto (->Calendar)
(.setTime (->Timestamp date)))))
(.setTime (->Timestamp date))))
(^java.util.Calendar [date, ^String timezone-id]
(doto (->Calendar date)
(.setTimeZone (TimeZone/getTimeZone timezone-id)))))
(defn relative-date
"Return a new `Timestamp` relative to the current time using a relative date UNIT.
......@@ -173,9 +179,11 @@
(date-extract :year) -> 2015"
([unit]
(date-extract unit (System/currentTimeMillis)))
(date-extract unit (System/currentTimeMillis) "UTC"))
([unit date]
(let [cal (->Calendar date)]
(date-extract unit date "UTC"))
([unit date timezone-id]
(let [cal (->Calendar date timezone-id)]
(case unit
:minute-of-hour (.get cal Calendar/MINUTE)
:hour-of-day (.get cal Calendar/HOUR_OF_DAY)
......@@ -186,7 +194,7 @@
;; 1 = First week of year
:week-of-year (.get cal Calendar/WEEK_OF_YEAR)
:month-of-year (inc (.get cal Calendar/MONTH))
:quarter-of-year (let [month (date-extract :month-of-year date)]
:quarter-of-year (let [month (date-extract :month-of-year date timezone-id)]
(int (/ (+ 2 month)
3)))
:year (.get cal Calendar/YEAR)))))
......@@ -195,45 +203,61 @@
(def ^:private ^:const date-trunc-units
#{:minute :hour :day :week :month :quarter})
(defn- trunc-with-format [format-string date timezone-id]
(->Timestamp (format-date (time/with-zone (time/formatter format-string)
(t/time-zone-for-id timezone-id))
date)))
(defn- trunc-with-floor [date amount-ms]
(->Timestamp (* (math/floor (/ (.getTime (->Timestamp date))
amount-ms))
amount-ms)))
(defn- ->first-day-of-week [date timezone-id]
(let [day-of-week (date-extract :day-of-week date timezone-id)]
(relative-date :day (- (dec day-of-week)) date)))
(defn- format-string-for-quarter ^String [date timezone-id]
(let [year (date-extract :year date timezone-id)
quarter (date-extract :quarter-of-year date timezone-id)
month (- (* 3 quarter) 2)]
(format "%d-%02d-01ZZ" year month)))
(defn date-trunc
"Truncate DATE to UNIT. DATE defaults to now.
(date-trunc :month).
;; -> #inst \"2015-11-01T00:00:00\""
(^java.sql.Timestamp [unit]
(date-trunc unit (System/currentTimeMillis)))
(date-trunc unit (System/currentTimeMillis) "UTC"))
(^java.sql.Timestamp [unit date]
(let [trunc-with-format (fn trunc-with-format
([format-string]
(trunc-with-format format-string date))
([format-string d]
(->Timestamp (format-date format-string d))))]
(case unit
:minute (trunc-with-format "yyyy-MM-dd'T'HH:mm:00+00:00")
:hour (trunc-with-format "yyyy-MM-dd'T'HH:00:00+00:00")
:day (trunc-with-format "yyyy-MM-dd+00:00")
:week (let [day-of-week (date-extract :day-of-week date)
date (relative-date :day (- (dec day-of-week)) date)]
(trunc-with-format "yyyy-MM-dd+00:00" date))
:month (trunc-with-format "yyyy-MM-01+00:00")
:quarter (let [year (date-extract :year date)
quarter (date-extract :quarter-of-year date)]
(->Timestamp (format "%d-%02d-01+00:00" year (- (* 3 quarter)
2))))))))
(date-trunc unit date "UTC"))
(^java.sql.Timestamp [unit date timezone-id]
(case unit
;; For minute and hour truncation timezone should not be taken into account
:minute (trunc-with-floor date (* 60 1000))
:hour (trunc-with-floor date (* 60 60 1000))
:day (trunc-with-format "yyyy-MM-ddZZ" date timezone-id)
:week (trunc-with-format "yyyy-MM-ddZZ" (->first-day-of-week date timezone-id) timezone-id)
:month (trunc-with-format "yyyy-MM-01ZZ" date timezone-id)
:quarter (trunc-with-format (format-string-for-quarter date timezone-id) date timezone-id))))
(defn date-trunc-or-extract
"Apply date bucketing with UNIT to DATE. DATE defaults to now."
([unit]
(date-trunc-or-extract unit (System/currentTimeMillis)))
(date-trunc-or-extract unit (System/currentTimeMillis) "UTC"))
([unit date]
(date-trunc-or-extract unit date "UTC"))
([unit date timezone-id]
(cond
(= unit :default) date
(contains? date-extract-units unit)
(date-extract unit date)
(date-extract unit date timezone-id)
(contains? date-trunc-units unit)
(date-trunc unit date))))
(date-trunc unit date timezone-id))))
(defn format-nanoseconds
"Format a time interval in nanoseconds to something more readable (µs/ms/etc.)
......
test/metabase/util_test.clj
+ 70
− 30
View file @ 5dba8471
......@@ -6,45 +6,85 @@
;;; Date stuff
(def ^:private ^:const friday-the-13th #inst "2015-11-13T19:05:55")
(def ^:private ^:const saturday-the-14th #inst "2015-11-14T04:18:26")
(def ^:private ^:const saturday-the-31st #inst "2005-12-31T19:05:55")
(def ^:private ^:const sunday-the-1st #inst "2006-01-01T04:18:26")
(expect false (is-temporal? nil))
(expect false (is-temporal? 123))
(expect false (is-temporal? "abc"))
(expect false (is-temporal? [1 2 3]))
(expect false (is-temporal? {:a "b"}))
(expect true (is-temporal? friday-the-13th))
(expect true (is-temporal? saturday-the-31st))
(expect friday-the-13th (->Timestamp (->Date friday-the-13th)))
(expect friday-the-13th (->Timestamp (->Calendar friday-the-13th)))
(expect friday-the-13th (->Timestamp (->Calendar (.getTime friday-the-13th))))
(expect friday-the-13th (->Timestamp (.getTime friday-the-13th)))
(expect friday-the-13th (->Timestamp "2015-11-13T19:05:55+00:00"))
(expect saturday-the-31st (->Timestamp (->Date saturday-the-31st)))
(expect saturday-the-31st (->Timestamp (->Calendar saturday-the-31st)))
(expect saturday-the-31st (->Timestamp (->Calendar (.getTime saturday-the-31st))))
(expect saturday-the-31st (->Timestamp (.getTime saturday-the-31st)))
(expect saturday-the-31st (->Timestamp "2005-12-31T19:05:55+00:00"))
(expect nil (->iso-8601-datetime nil nil))
(expect "2015-11-13T19:05:55.000Z" (->iso-8601-datetime friday-the-13th nil))
(expect "2015-11-13T11:05:55.000-08:00" (->iso-8601-datetime friday-the-13th "US/Pacific"))
(expect "2015-11-14T04:05:55.000+09:00" (->iso-8601-datetime friday-the-13th "Asia/Tokyo"))
(expect 5 (date-extract :minute-of-hour friday-the-13th))
(expect 19 (date-extract :hour-of-day friday-the-13th))
(expect 6 (date-extract :day-of-week friday-the-13th))
(expect 7 (date-extract :day-of-week saturday-the-14th))
(expect 13 (date-extract :day-of-month friday-the-13th))
(expect 317 (date-extract :day-of-year friday-the-13th))
(expect 46 (date-extract :week-of-year friday-the-13th))
(expect 11 (date-extract :month-of-year friday-the-13th))
(expect 4 (date-extract :quarter-of-year friday-the-13th))
(expect 2015 (date-extract :year friday-the-13th))
(expect #inst "2015-11-13T19:05" (date-trunc :minute friday-the-13th))
(expect #inst "2015-11-13T19:00" (date-trunc :hour friday-the-13th))
(expect #inst "2015-11-13" (date-trunc :day friday-the-13th))
(expect #inst "2015-11-08" (date-trunc :week friday-the-13th))
(expect #inst "2015-11-08" (date-trunc :week saturday-the-14th))
(expect #inst "2015-11-01" (date-trunc :month friday-the-13th))
(expect #inst "2015-10-01" (date-trunc :quarter friday-the-13th))
(expect "2005-12-31T19:05:55.000Z" (->iso-8601-datetime saturday-the-31st nil))
(expect "2005-12-31T11:05:55.000-08:00" (->iso-8601-datetime saturday-the-31st "US/Pacific"))
(expect "2006-01-01T04:05:55.000+09:00" (->iso-8601-datetime saturday-the-31st "Asia/Tokyo"))
(expect 5 (date-extract :minute-of-hour saturday-the-31st "UTC"))
(expect 19 (date-extract :hour-of-day saturday-the-31st "UTC"))
(expect 7 (date-extract :day-of-week saturday-the-31st "UTC"))
(expect 1 (date-extract :day-of-week sunday-the-1st "UTC"))
(expect 31 (date-extract :day-of-month saturday-the-31st "UTC"))
(expect 365 (date-extract :day-of-year saturday-the-31st "UTC"))
(expect 53 (date-extract :week-of-year saturday-the-31st "UTC"))
(expect 12 (date-extract :month-of-year saturday-the-31st "UTC"))
(expect 4 (date-extract :quarter-of-year saturday-the-31st "UTC"))
(expect 2005 (date-extract :year saturday-the-31st "UTC"))
(expect 5 (date-extract :minute-of-hour saturday-the-31st "US/Pacific"))
(expect 11 (date-extract :hour-of-day saturday-the-31st "US/Pacific"))
(expect 7 (date-extract :day-of-week saturday-the-31st "US/Pacific"))
(expect 7 (date-extract :day-of-week sunday-the-1st "US/Pacific"))
(expect 31 (date-extract :day-of-month saturday-the-31st "US/Pacific"))
(expect 365 (date-extract :day-of-year saturday-the-31st "US/Pacific"))
(expect 53 (date-extract :week-of-year saturday-the-31st "US/Pacific"))
(expect 12 (date-extract :month-of-year saturday-the-31st "US/Pacific"))
(expect 4 (date-extract :quarter-of-year saturday-the-31st "US/Pacific"))
(expect 2005 (date-extract :year saturday-the-31st "US/Pacific"))
(expect 5 (date-extract :minute-of-hour saturday-the-31st "Asia/Tokyo"))
(expect 4 (date-extract :hour-of-day saturday-the-31st "Asia/Tokyo"))
(expect 1 (date-extract :day-of-week saturday-the-31st "Asia/Tokyo"))
(expect 1 (date-extract :day-of-week sunday-the-1st "Asia/Tokyo"))
(expect 1 (date-extract :day-of-month saturday-the-31st "Asia/Tokyo"))
(expect 1 (date-extract :day-of-year saturday-the-31st "Asia/Tokyo"))
(expect 1 (date-extract :week-of-year saturday-the-31st "Asia/Tokyo"))
(expect 1 (date-extract :month-of-year saturday-the-31st "Asia/Tokyo"))
(expect 1 (date-extract :quarter-of-year saturday-the-31st "Asia/Tokyo"))
(expect 2006 (date-extract :year saturday-the-31st "Asia/Tokyo"))
(expect #inst "2005-12-31T19:05" (date-trunc :minute saturday-the-31st "UTC"))
(expect #inst "2005-12-31T19:00" (date-trunc :hour saturday-the-31st "UTC"))
(expect #inst "2005-12-31" (date-trunc :day saturday-the-31st "UTC"))
(expect #inst "2005-12-25" (date-trunc :week saturday-the-31st "UTC"))
(expect #inst "2006-01-01" (date-trunc :week sunday-the-1st "UTC"))
(expect #inst "2005-12-01" (date-trunc :month saturday-the-31st "UTC"))
(expect #inst "2005-10-01" (date-trunc :quarter saturday-the-31st "UTC"))
(expect #inst "2005-12-31T19:05" (date-trunc :minute saturday-the-31st "Asia/Tokyo"))
(expect #inst "2005-12-31T19:00" (date-trunc :hour saturday-the-31st "Asia/Tokyo"))
(expect #inst "2006-01-01+09:00" (date-trunc :day saturday-the-31st "Asia/Tokyo"))
(expect #inst "2006-01-01+09:00" (date-trunc :week saturday-the-31st "Asia/Tokyo"))
(expect #inst "2006-01-01+09:00" (date-trunc :week sunday-the-1st "Asia/Tokyo"))
(expect #inst "2006-01-01+09:00" (date-trunc :month saturday-the-31st "Asia/Tokyo"))
(expect #inst "2006-01-01+09:00" (date-trunc :quarter saturday-the-31st "Asia/Tokyo"))
(expect #inst "2005-12-31T19:05" (date-trunc :minute saturday-the-31st "US/Pacific"))
(expect #inst "2005-12-31T19:00" (date-trunc :hour saturday-the-31st "US/Pacific"))
(expect #inst "2005-12-31-08:00" (date-trunc :day saturday-the-31st "US/Pacific"))
(expect #inst "2005-12-25-08:00" (date-trunc :week saturday-the-31st "US/Pacific"))
(expect #inst "2005-12-25-08:00" (date-trunc :week sunday-the-1st "US/Pacific"))
(expect #inst "2005-12-01-08:00" (date-trunc :month saturday-the-31st "US/Pacific"))
(expect #inst "2005-10-01-08:00" (date-trunc :quarter saturday-the-31st "US/Pacific"))
;;; ## tests for HOST-UP?
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment