Skip to content
Snippets Groups Projects
Commit 74dbfb85 authored by Cam Saul's avatar Cam Saul
Browse files

test fix

parent 18c6cc40
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Showing
with 38 additions and 142 deletions
src/metabase/driver/generic_sql/query_processor.clj
+ 3
− 7
View file @ 74dbfb85
......@@ -151,13 +151,9 @@
[_ field-id & _] {(field-id->kw field-id)
;; If the field in question is a date field we need to cast the YYYY-MM-DD string that comes back from the UI to a SQL date
(let [cast-value-if-needed (fn [v]
(cond (or (= (type v) java.sql.Date)
(= (type v) java.util.Date)) `(raw ~(format "CAST('%s' AS DATE)" (.toString ^java.sql.Date v)))
(not (string? v)) v
(date-field-id? field-id) (u/parse-date-yyyy-mm-dd v)
(= (field-id->special-type field-id)
:timestamp_seconds) (u/date-yyyy-mm-dd->unix-timestamp v)
:else v))]
(if-not (or (= (type v) java.sql.Date)
(= (type v) java.util.Date)) v
`(raw ~(format "CAST('%s' AS DATE)" (.toString ^java.sql.Date v)))))]
(match subclause
["NOT_NULL" _] ['not= nil]
["IS_NULL" _] ['= nil]
......
src/metabase/driver/generic_sql/util.clj
+ 6
− 8
View file @ 74dbfb85
......@@ -97,7 +97,6 @@
[field-name base-type special-type]
{:pre [(string? field-name)
(keyword? base-type)]}
;; TODO - timestamp stuff is Postgres-specific
(cond
(contains? #{:DateField :DateTimeField} base-type) `(korma/raw ~(format "CAST(\"%s\" AS DATE)" field-name))
(= special-type :timestamp_seconds) `(korma/raw ~((:cast-timestamp-seconds-field-to-date-fn qp/*driver*) field-name))
......@@ -115,14 +114,13 @@
(contains? #{:DateField :DateTimeField} field-base-type) (format "CAST(%s AS DATE)" field-name)
:else field-name)))
(def field-id->kw
(defn field-id->kw
"Given a metabase `Field` ID, return a keyword for use in the Korma form (or a casted raw string for date fields)."
(memoize ; This can be memozied since the names and base_types of Fields never change
(fn [field-id] ; * if a field is renamed the old field will just be marked as `inactive` and a new Field will be created
{:pre [(integer? field-id)]} ; * if a field's type *actually* changes we have no logic in driver.generic-sql.sync to handle that case any way (TODO - fix issue?)
(if-let [{field-name :name, field-type :base_type, special-type :special_type} (sel :one [Field :name :base_type :special_type] :id field-id)]
(castify-field field-name field-type special-type)
(throw (Exception. (format "Field with ID %d doesn't exist!" field-id)))))))
[field-id]
{:pre [(integer? field-id)]}
(if-let [{field-name :name, field-type :base_type, special-type :special_type} (sel :one [Field :name :base_type :special_type] :id field-id)]
(castify-field field-name field-type special-type)
(throw (Exception. (format "Field with ID %d doesn't exist!" field-id)))))
(def date-field-id?
"Does FIELD-ID correspond to a field that is a Date?"
......
src/metabase/driver/query_processor.clj
+ 0
− 34
View file @ 74dbfb85
......@@ -16,7 +16,6 @@
add-implicit-limit
add-implicit-fields
get-special-column-info
preprocess-rewrite-timestamp-equals-filter
preprocess-cumulative-sum
preprocess-structured
remove-empty-clauses)
......@@ -76,7 +75,6 @@
add-implicit-breakout-order-by
add-implicit-limit
add-implicit-fields
preprocess-rewrite-timestamp-equals-filter
preprocess-cumulative-sum))]
(when-not *disable-qp-logging*
(log/debug (colorize.core/cyan "\n******************** PREPROCESSED: ********************\n"
......@@ -150,38 +148,6 @@
(assoc query :fields (sel :many :id Field :table_id source_table, :active true, :preview_display true,
:field_type [not= "sensitive"], (order :position :asc), (order :id :desc))))))
(def ^:private ^:const seconds-per-day (* 24 60 60))
(def ^:private ^:const milliseconds-per-day (* seconds-per-day 1000))
(defn- rewrite-timestamp-filter=
"Rewrite an `=` filter clause for a timestamp `Field`. "
[{:keys [field], {timestamp :value, special-type :special-type, :as value} :value}]
;; The timestamps we create 00:00 on the day in question, re-write the filter as a ["BETWEEN" field timestamp (+ timestamp 1-day)]
(expand/map->Filter:Between {:type :between
:field field
:min value
:max (expand/map->Value (assoc value
:value (+ timestamp (case special-type
:timestamp_seconds seconds-per-day
:timestamp_milliseconds milliseconds-per-day))))}))
(defn preprocess-rewrite-timestamp-equals-filter
"In order for `=` filter clauses to work with timestamps (allowing the user to match a given day) we need to rewrite them as
`BETWEEN` clauses. Check and see if the `filter` clause contains any subclauses that fit the bill and rewrite them accordingly."
[query]
(if-not (:filter query)
;; If there's no filter clause there's nothing to do
query
;; Otherwise rewrite as needed
(assoc query :filter (-> (:filter *expanded-query*)
(update-in [:subclauses] #(for [{:keys [filter-type], {:keys [special-type]} :field, :as subclause} %]
(if (and (= filter-type :=)
(contains? #{:timestamp_seconds
:timestamp_milliseconds} special-type))
(rewrite-timestamp-filter= subclause)
subclause)))
expand/collapse))))
;; ### PREPROCESS-CUMULATIVE-SUM
......
src/metabase/driver/query_processor/expand.clj
+ 5
− 5
View file @ 74dbfb85
......@@ -156,11 +156,11 @@
[{:keys [value base-type special-type] :as qp-value}]
(assoc qp-value
:original-value value
:value (or (when (string? value) ; Since Value *doesn't* revert to YYYY-MM-DD when collapsing make sure we're not parsing it twice
(cond
(contains? #{:DateField :DateTimeField} base-type) (u/parse-date-yyyy-mm-dd value)
(= special-type :timestamp_seconds) (u/date-yyyy-mm-dd->unix-timestamp value)
(= special-type :timestamp_milliseconds) (* (u/date-yyyy-mm-dd->unix-timestamp value) 1000)))
;; Since Value *doesn't* revert to YYYY-MM-DD when collapsing make sure we're not parsing it twice
:value (or (when (and (string? value)
(or (contains? #{:DateField :DateTimeField} base-type)
(contains? #{:timestamp_seconds :timestamp_milliseconds} special-type)))
(u/parse-date-yyyy-mm-dd value))
value)))
;; Replace values with these during first pass over Query.
......
test/metabase/driver/query_processor_test.clj
+ 21
− 65
View file @ 74dbfb85
......@@ -778,71 +778,23 @@
;; ## Unix timestamp special type fields <3
;; There are 4 events in us-history-1607-to-1774 whose year is < 1765
(datasets/expect-with-all-datasets 4
(with-temp-db [db (dataset-loader) defs/us-history-1607-to-1774]
(-> (driver/process-query {:database (:id db)
:type :query
:query {:source_table (:id &events)
:aggregation ["count"]
:filter ["<" (:id &events.timestamp) "1765-01-01"]}})
:data
:rows
first
first)))
;; Check that Timestamps are getting converted the way we expect
(datasets/expect-with-all-datasets
["1607-05-14"
"1620-11-11"
"1752-06-15"
"1754-05-28"
"1765-03-22"
"1765-03-24"
"1765-10-19"
"1766-03-18"
"1767-06-29"
"1770-03-05"
"1773-05-10"
"1773-12-16"
"1774-03-31"
"1774-09-05"]
(with-temp-db [db (dataset-loader) defs/us-history-1607-to-1774]
(->> (driver/process-query {:database (:id db)
:type "query"
:query {:source_table (:id &events)
:aggregation ["rows"]}})
:data
:rows
(map last))))
;;; Unix timestamp breakouts -- SQL only
;; There were 9 "sad toucan incidents" on 2015-06-02
(datasets/expect-with-datasets #{:generic-sql}
[["1607-05-14" (->sum-type 1)]
["1620-11-11" (->sum-type 3)]
["1752-06-15" (->sum-type 7)]
["1754-05-28" (->sum-type 10)]
["1765-03-22" (->sum-type 15)]
["1765-03-24" (->sum-type 21)]
["1765-10-19" (->sum-type 28)]
["1766-03-18" (->sum-type 36)]
["1767-06-29" (->sum-type 45)]
["1770-03-05" (->sum-type 55)]
["1773-05-10" (->sum-type 66)]
["1773-12-16" (->sum-type 78)]
["1774-03-31" (->sum-type 91)]
["1774-09-05" (->sum-type 105)]]
(with-temp-db [db (dataset-loader) defs/us-history-1607-to-1774]
9
(with-temp-db [db (dataset-loader) defs/sad-toucan-incidents]
(->> (driver/process-query {:database (:id db)
:type "query"
:query {:source_table (:id &events)
:aggregation ["cum_sum" (:id &events.id)]
:breakout [(:id &events.timestamp)]}})
:query {:source_table (:id &incidents)
:filter ["AND"
[">" (:id &incidents.timestamp) "2015-06-01"]
["<" (:id &incidents.timestamp) "2015-06-03"]]
:order_by [[(:id &incidents.timestamp) "ascending"]]}})
:data
:rows)))
:rows
count)))
;;; Unix timestamp breakouts -- SQL only
(datasets/expect-with-datasets #{:generic-sql}
[["2015-06-01" 6]
["2015-06-02" 9]
......@@ -855,9 +807,13 @@
["2015-06-09" 7]
["2015-06-10" 8]]
(with-temp-db [db (dataset-loader) defs/sad-toucan-incidents]
(driver/process-query {:database (:id db)
:type "query"
:query {:source_table (:id &incidents)
:aggregation ["count"]
:breakout [(:id &incidents.timestamp)]
:limit 10}})))
(->> (driver/process-query {:database (:id db)
:type "query"
:query {:source_table (:id &incidents)
:aggregation ["count"]
:breakout [(:id &incidents.timestamp)]
:limit 10}})
:data
:rows
(map (fn [[^java.util.Date date count]]
[(.toString date) (int count)])))))
test/metabase/test/data/dataset_definitions.clj
+ 3
− 23
View file @ 74dbfb85
......@@ -5,7 +5,7 @@
;; ## Helper Functions
(defn unix-timestamp-ms
(defn- unix-timestamp-ms
"Create a Unix timestamp (in milliseconds).
(unix-timestamp-ms :year 2012 :month 12 :date 27)"
......@@ -16,33 +16,13 @@
long)) ; coerce to long since Korma doesn't know how to insert bigints
(defn unix-timestamp
(defn- unix-timestamp
"Create a Unix timestamp, in seconds."
^Long [& args]
(/ (apply unix-timestamp-ms args) 1000))
;; ## Datasets
(def-database-definition us-history-1607-to-1774
["events" [{:field-name "name"
:base-type :CharField}
{:field-name "timestamp"
:base-type :BigIntegerField
:special-type :timestamp_seconds}]
[["Jamestown Settlement Founded" (unix-timestamp :year 1607 :month 5 :date 14)]
["Mayflower Compact Signed" (unix-timestamp :year 1620 :month 11 :date 11)]
["Ben Franklin's Kite Experiment" (unix-timestamp :year 1752 :month 6 :date 15)]
["French and Indian War Begins" (unix-timestamp :year 1754 :month 5 :date 28)]
["Stamp Act Enacted" (unix-timestamp :year 1765 :month 3 :date 22)]
["Quartering Act Enacted" (unix-timestamp :year 1765 :month 3 :date 24)]
["Stamp Act Congress Meets" (unix-timestamp :year 1765 :month 10 :date 19)]
["Stamp Act Repealed" (unix-timestamp :year 1766 :month 3 :date 18)]
["Townshend Acts Passed" (unix-timestamp :year 1767 :month 6 :date 29)]
["Boston Massacre" (unix-timestamp :year 1770 :month 3 :date 5)]
["Tea Act Passed" (unix-timestamp :year 1773 :month 5 :date 10)]
["Boston Tea Party" (unix-timestamp :year 1773 :month 12 :date 16)]
["Boston Port Act Passed" (unix-timestamp :year 1774 :month 3 :date 31)]
["First Continental Congress Held" (unix-timestamp :year 1774 :month 9 :date 5)]]])
;; ## Datasets
(def ^:private ^:const edn-definitions-dir "./test/metabase/test/data/dataset_definitions/")
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment