Skip to content
Snippets Groups Projects
Commit e30c6d1d authored by Tom Robinson's avatar Tom Robinson
Browse files

Merge branch 'master' of github.com:metabase/metabase into saved-questions

parents cb18ec0a e153fbde
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Showing
with 443 additions and 188 deletions
.dir-locals.el
+ 0
− 2
View file @ e30c6d1d
...@@ -15,8 +15,6 @@ ...@@ -15,8 +15,6 @@
(check 1) (check 1)
(checkp 1) (checkp 1)
(cond-as-> 2) (cond-as-> 2)
(cond-let 0)
(conda 0)
(context 2) (context 2)
(create-database-definition 1) (create-database-definition 1)
(engine-case 0) (engine-case 0)
......
src/metabase/driver.clj
+ 3
− 2
View file @ e30c6d1d
...@@ -484,10 +484,11 @@ ...@@ -484,10 +484,11 @@
"Save QueryExecution state and construct a completed (successful) query response" "Save QueryExecution state and construct a completed (successful) query response"
[query-execution query-result] [query-execution query-result]
;; record our query execution and format response ;; record our query execution and format response
(-> (u/assoc<> query-execution (-> (assoc query-execution
:status :completed :status :completed
:finished_at (u/new-sql-timestamp) :finished_at (u/new-sql-timestamp)
:running_time (- (System/currentTimeMillis) (:start_time_millis <>)) :running_time (- (System/currentTimeMillis)
(:start_time_millis query-execution))
:result_rows (get query-result :row_count 0)) :result_rows (get query-result :row_count 0))
(dissoc :start_time_millis) (dissoc :start_time_millis)
(save-query-execution) (save-query-execution)
......
src/metabase/driver/query_processor.clj
+ 67
− 27
View file @ e30c6d1d
...@@ -13,7 +13,7 @@ ...@@ -13,7 +13,7 @@
[interface :refer :all] [interface :refer :all]
[macros :as macros] [macros :as macros]
[resolve :as resolve]) [resolve :as resolve])
[metabase.models.field :refer [Field], :as field] [metabase.models.field :refer [Field]]
[metabase.util :as u]) [metabase.util :as u])
(:import (schema.utils NamedError ValidationError))) (:import (schema.utils NamedError ValidationError)))
...@@ -34,9 +34,9 @@ ...@@ -34,9 +34,9 @@
false) false)
;; +----------------------------------------------------------------------------------------------------+ ;;; +----------------------------------------------------------------------------------------------------+
;; | QP INTERNAL IMPLEMENTATION | ;;; | QP INTERNAL IMPLEMENTATION |
;; +----------------------------------------------------------------------------------------------------+ ;;; +----------------------------------------------------------------------------------------------------+
(defn structured-query? (defn structured-query?
...@@ -92,6 +92,12 @@ ...@@ -92,6 +92,12 @@
msg))) msg)))
explanation)))) explanation))))
;;; +-----------------------------------------------------------------------------------------------------------------+
;;; | MIDDLEWARE FUNCTIONS |
;;; +-----------------------------------------------------------------------------------------------------------------+
(defn- wrap-catch-exceptions [qp] (defn- wrap-catch-exceptions [qp]
(fn [query] (fn [query]
(try (qp query) (try (qp query)
...@@ -104,28 +110,47 @@ ...@@ -104,28 +110,47 @@
(fail query e))))) (fail query e)))))
(defn- pre-add-settings [qp] (defn- pre-add-settings
"Adds the `:settings` map to the query which can contain any fixed properties that would be useful at execution time.
Currently supports a settings object like:
{:report-timezone \"US/Pacific\"}
"
[qp]
(fn [{:keys [driver] :as query}] (fn [{:keys [driver] :as query}]
(let [settings {:report-timezone (when (driver/driver-supports? driver :set-timezone) (let [settings {:report-timezone (when (driver/driver-supports? driver :set-timezone)
(let [report-tz (driver/report-timezone)] (let [report-tz (driver/report-timezone)]
(when-not (empty? report-tz) (when-not (empty? report-tz)
report-tz)))}] report-tz)))}]
(->> (u/filter-nil-values settings) (qp (assoc query :settings (m/filter-vals (complement nil?) settings))))))
(assoc query :settings)
qp))))
(defn- pre-expand [qp] (defn- pre-expand-macros
"Looks for macros in a structured (unexpanded) query and substitutes the macros for their contents."
[qp]
(fn [query] (fn [query]
(qp (if (structured-query? query) ;; if necessary, handle macro substitution
(let [macro-expanded-query (macros/expand-macros query)] (let [query (if-not (structured-query? query)
(when (and (not *disable-qp-logging*) query
(not= macro-expanded-query query)) ;; for structured queries run our macro expansion
(log/debug (u/format-color 'cyan "\n\nMACRO/SUBSTITUTED: 😻\n%s" (u/pprint-to-str macro-expanded-query)))) (u/prog1 (macros/expand-macros query)
(-> macro-expanded-query (when (and (not *disable-qp-logging*)
expand/expand (not= <> query))
resolve/resolve)) (log/debug (u/format-color 'cyan "\n\nMACRO/SUBSTITUTED: 😻\n%s" (u/pprint-to-str <>))))))]
query)))) (qp query))))
(defn- pre-expand-resolve
"Transforms an MBQL into an expanded form with more information and structure. Also resolves references to fields, tables,
etc, into their concrete details which are necessary for query formation by the executing driver."
[qp]
(fn [query]
;; if necessary, expand/resolve the query
(let [query (if-not (structured-query? query)
query
;; for structured queries we expand first, then resolve
(resolve/resolve (expand/expand query)))]
(qp query))))
(defn- post-add-row-count-and-status (defn- post-add-row-count-and-status
...@@ -144,6 +169,7 @@ ...@@ -144,6 +169,7 @@
;; Add :rows_truncated if we've hit the limit so the UI can let the user know ;; Add :rows_truncated if we've hit the limit so the UI can let the user know
(= num-results results-limit) (assoc-in [:data :rows_truncated] results-limit))))) (= num-results results-limit) (assoc-in [:data :rows_truncated] results-limit)))))
(defn- format-rows [{:keys [report-timezone]} rows] (defn- format-rows [{:keys [report-timezone]} rows]
(for [row rows] (for [row rows]
(for [v row] (for [v row]
...@@ -286,6 +312,18 @@ ...@@ -286,6 +312,18 @@
absolute-max-results)))))) absolute-max-results))))))
(defn post-annotate
"QP middleware that runs directly after the the query is run and adds metadata as appropriate."
[qp]
(fn [query]
(if-not (structured-query? query)
;; non-structured queries are not affected
(qp query)
;; for structured queries capture the results and annotate
(let [results (qp query)]
(annotate/annotate query results)))))
(defn- pre-log-query [qp] (defn- pre-log-query [qp]
(fn [query] (fn [query]
(when (and (structured-query? query) (when (and (structured-query? query)
...@@ -314,9 +352,9 @@ ...@@ -314,9 +352,9 @@
(qp query)))) (qp query))))
;; +------------------------------------------------------------------------------------------------------------------------+ ;;; +-------------------------------------------------------------------------------------------------------+
;; | QUERY PROCESSOR | ;;; | QUERY PROCESSOR |
;; +------------------------------------------------------------------------------------------------------------------------+ ;;; +-------------------------------------------------------------------------------------------------------+
;; The way these functions are applied is actually straight-forward; it matches the middleware pattern used by Compojure. ;; The way these functions are applied is actually straight-forward; it matches the middleware pattern used by Compojure.
...@@ -351,20 +389,22 @@ ...@@ -351,20 +389,22 @@
(when-not *disable-qp-logging* (when-not *disable-qp-logging*
(log/debug (u/format-color 'blue "\nQUERY: 😎\n%s" (u/pprint-to-str query)))) (log/debug (u/format-color 'blue "\nQUERY: 😎\n%s" (u/pprint-to-str query))))
(binding [*driver* driver] (binding [*driver* driver]
(let [driver-process-query (partial (if (structured-query? query) (let [driver-process-in-context (partial driver/process-query-in-context driver)
driver/process-structured driver-process-query (partial (if (structured-query? query)
driver/process-native) driver)] driver/process-structured
driver/process-native) driver)]
((<<- wrap-catch-exceptions ((<<- wrap-catch-exceptions
pre-add-settings pre-add-settings
pre-expand pre-expand-macros
(driver/process-query-in-context driver) pre-expand-resolve
driver-process-in-context
post-add-row-count-and-status post-add-row-count-and-status
post-format-rows post-format-rows
pre-add-implicit-fields pre-add-implicit-fields
pre-add-implicit-breakout-order-by pre-add-implicit-breakout-order-by
cumulative-sum cumulative-sum
limit limit
annotate/post-annotate post-annotate
pre-log-query pre-log-query
wrap-guard-multiple-calls wrap-guard-multiple-calls
driver-process-query) (assoc query :driver driver))))) driver-process-query) (assoc query :driver driver)))))
src/metabase/driver/query_processor/annotate.clj
+ 11
− 16
View file @ e30c6d1d
...@@ -234,23 +234,18 @@ ...@@ -234,23 +234,18 @@
(m/distinct-by :name) (m/distinct-by :name)
add-extra-info-to-fk-fields))) add-extra-info-to-fk-fields)))
(defn post-annotate (defn annotate
"QP middleware that runs directly after the the query is ran. This stage: "Post-process a structured query to add metadata to the results. This stage:
1. Sorts the results according to the rules at the top of this page 1. Sorts the results according to the rules at the top of this page
2. Resolves the Fields returned in the results and adds information like `:columns` and `:cols` 2. Resolves the Fields returned in the results and adds information like `:columns` and `:cols`
expected by the frontend." expected by the frontend."
[qp] [query results]
(fn [query] (let [result-keys (set (keys (first results)))
(if (= :query (keyword (:type query))) cols (resolve-sort-and-format-columns (:query query) result-keys)
(let [results (qp query) columns (mapv :name cols)]
result-keys (set (keys (first results))) {:cols (vec (for [col cols]
cols (resolve-sort-and-format-columns (:query query) result-keys) (update col :name name)))
columns (mapv :name cols)] :columns (mapv name columns)
{:cols (vec (for [col cols] :rows (for [row results]
(update col :name name))) (mapv row columns))}))
:columns (mapv name columns)
:rows (for [row results]
(mapv row columns))})
;; for non-structured queries we do nothing
(qp query))))
src/metabase/models/metric.clj
+ 7
− 6
View file @ e30c6d1d
(ns metabase.models.metric (ns metabase.models.metric
(:require [korma.core :as k] (:require [korma.core :as k]
[medley.core :as m]
[metabase.db :as db] [metabase.db :as db]
[metabase.events :as events] [metabase.events :as events]
(metabase.models [dependency :as dependency] (metabase.models [dependency :as dependency]
...@@ -30,17 +31,17 @@ ...@@ -30,17 +31,17 @@
(defn- diff-metrics [this metric1 metric2] (defn- diff-metrics [this metric1 metric2]
(if-not metric1 (if-not metric1
;; this is the first version of the metric ;; this is the first version of the metric
(u/update-values (select-keys metric2 [:name :description :definition]) (fn [v] {:after v})) (m/map-vals (fn [v] {:after v}) (select-keys metric2 [:name :description :definition]))
;; do our diff logic ;; do our diff logic
(let [base-diff (revision/default-diff-map this (let [base-diff (revision/default-diff-map this
(select-keys metric1 [:name :description :definition]) (select-keys metric1 [:name :description :definition])
(select-keys metric2 [:name :description :definition]))] (select-keys metric2 [:name :description :definition]))]
(cond-> (merge-with merge (cond-> (merge-with merge
(u/update-values (:after base-diff) (fn [v] {:after v})) (m/map-vals (fn [v] {:after v}) (:after base-diff))
(u/update-values (:before base-diff) (fn [v] {:before v}))) (m/map-vals (fn [v] {:before v}) (:before base-diff)))
(or (get-in base-diff [:after :definition]) (or (get-in base-diff [:after :definition])
(get-in base-diff [:before :definition])) (assoc :definition {:before (get-in metric1 [:definition]) (get-in base-diff [:before :definition])) (assoc :definition {:before (get-in metric1 [:definition])
:after (get-in metric2 [:definition])}))))) :after (get-in metric2 [:definition])})))))
(u/strict-extend (class Metric) (u/strict-extend (class Metric)
revision/IRevisioned revision/IRevisioned
......
src/metabase/models/segment.clj
+ 4
− 3
View file @ e30c6d1d
(ns metabase.models.segment (ns metabase.models.segment
(:require [korma.core :as k] (:require [korma.core :as k]
[medley.core :as m]
[metabase.db :as db] [metabase.db :as db]
[metabase.events :as events] [metabase.events :as events]
(metabase.models [hydrate :refer [hydrate]] (metabase.models [hydrate :refer [hydrate]]
...@@ -29,14 +30,14 @@ ...@@ -29,14 +30,14 @@
(defn- diff-segments [this segment1 segment2] (defn- diff-segments [this segment1 segment2]
(if-not segment1 (if-not segment1
;; this is the first version of the segment ;; this is the first version of the segment
(u/update-values (select-keys segment2 [:name :description :definition]) (fn [v] {:after v})) (m/map-vals (fn [v] {:after v}) (select-keys segment2 [:name :description :definition]))
;; do our diff logic ;; do our diff logic
(let [base-diff (revision/default-diff-map this (let [base-diff (revision/default-diff-map this
(select-keys segment1 [:name :description :definition]) (select-keys segment1 [:name :description :definition])
(select-keys segment2 [:name :description :definition]))] (select-keys segment2 [:name :description :definition]))]
(cond-> (merge-with merge (cond-> (merge-with merge
(u/update-values (:after base-diff) (fn [v] {:after v})) (m/map-vals (fn [v] {:after v}) (:after base-diff))
(u/update-values (:before base-diff) (fn [v] {:before v}))) (m/map-vals (fn [v] {:before v}) (:before base-diff)))
(or (get-in base-diff [:after :definition]) (or (get-in base-diff [:after :definition])
(get-in base-diff [:before :definition])) (assoc :definition {:before (get-in segment1 [:definition]) (get-in base-diff [:before :definition])) (assoc :definition {:before (get-in segment1 [:definition])
:after (get-in segment2 [:definition])}))))) :after (get-in segment2 [:definition])})))))
......
src/metabase/util.clj
+ 0
− 89
View file @ e30c6d1d
...@@ -114,10 +114,6 @@ ...@@ -114,10 +114,6 @@
(^String [date-format date] (^String [date-format date]
(time/unparse (->DateTimeFormatter date-format) (coerce/from-long (.getTime (->Timestamp date)))))) (time/unparse (->DateTimeFormatter date-format) (coerce/from-long (.getTime (->Timestamp date))))))
(def ^{:arglists '([] [date])} date->yyyy-mm-dd
"Format DATE as a `YYYY-MM-DD` string."
(partial format-date "yyyy-MM-dd"))
(def ^{:arglists '([] [date])} date->iso-8601 (def ^{:arglists '([] [date])} date->iso-8601
"Format DATE a an ISO-8601 string." "Format DATE a an ISO-8601 string."
(partial format-date :date-time)) (partial format-date :date-time))
...@@ -250,35 +246,6 @@ ...@@ -250,35 +246,6 @@
;;; ## Etc ;;; ## Etc
(defmacro assoc<>
"Like `assoc`, but associations happen sequentially; i.e. each successive binding can build
upon the result of the previous one using `<>`.
(assoc<> {}
:a 100
:b (+ 100 (:a <>)) ; -> {:a 100 :b 200}"
{:style/indent 1}
[object & kvs]
;; wrap in a `fn` so this can be used in `->`/`->>` forms
`((fn [~'<>]
(let [~@(apply concat (for [[k v] (partition 2 kvs)]
['<> `(assoc ~'<> ~k ~v)]))]
~'<>))
~object))
(defn format-num
"format a number into a more human readable form."
[number]
{:pre [(number? number)]}
(let [decimal-type? #(or (float? %) (decimal? %))]
(cond
;; looks like this is a decimal number, format with precision of 2
(and (decimal-type? number) (not (zero? (mod number 1)))) (format "%,.2f" number)
;; this is a decimal type number with no actual decimal value, so treat it as a whole number
(decimal-type? number) (format "%,d" (long number))
;; otherwise this is a whole number
:else (format "%,d" number))))
(defprotocol ^:private IClobToStr (defprotocol ^:private IClobToStr
(jdbc-clob->str ^String [this] (jdbc-clob->str ^String [this]
"Convert a Postgres/H2/SQLServer JDBC Clob to a string.")) "Convert a Postgres/H2/SQLServer JDBC Clob to a string."))
...@@ -337,19 +304,6 @@ ...@@ -337,19 +304,6 @@
(if (pred? (first args)) [(first args) (next args)] (if (pred? (first args)) [(first args) (next args)]
[default args])) [default args]))
;; provided courtesy of Jay Fields http://blog.jayfields.com/2011/08/clojure-apply-function-to-each-value-of.html
(defn update-values
"Update the values of a map by applying the given function.
Function expects the map value as an arg and optionally accepts additional args as passed."
[m f & args]
(reduce (fn [r [k v]] (assoc r k (apply f v args))) {} m))
(defn filter-nil-values
"Remove any keys from a MAP when the value is `nil`."
[m]
(into {} (for [[k v] m
:when (not (nil? v))]
{k v})))
(defn is-email? (defn is-email?
"Is STRING a valid email address?" "Is STRING a valid email address?"
...@@ -477,13 +431,6 @@ ...@@ -477,13 +431,6 @@
([color-symb x] ([color-symb x]
((ns-resolve 'colorize.core color-symb) (pprint-to-str x)))) ((ns-resolve 'colorize.core color-symb) (pprint-to-str x))))
(defmacro cond-let
"Like `if-let` or `when-let`, but for `cond`."
[binding-form then-form & more]
`(if-let ~binding-form ~then-form
~(when (seq more)
`(cond-let ~@more))))
(defn filtered-stacktrace (defn filtered-stacktrace
"Get the stack trace associated with E and return it as a vector with non-metabase frames filtered out." "Get the stack trace associated with E and return it as a vector with non-metabase frames filtered out."
[^Throwable e] [^Throwable e]
...@@ -538,42 +485,6 @@ ...@@ -538,42 +485,6 @@
[& body] [& body]
`(try ~@body (catch Throwable ~'_))) `(try ~@body (catch Throwable ~'_)))
(defn wrap-try-catch!
"Re-intern FN-SYMB as a new fn that wraps the original with a `try-catch`. Intended for debugging.
(defn z [] (throw (Exception. \"!\")))
(z) ; -> exception
(wrap-try-catch! 'z)
(z) ; -> nil; exception logged with log/error"
[fn-symb]
{:pre [(symbol? fn-symb)
(fn? @(resolve fn-symb))]}
(let [varr (resolve fn-symb)
{nmspc :ns, symb :name} (meta varr)]
(println (format "wrap-try-catch! %s/%s" nmspc symb))
(intern nmspc symb (wrap-try-catch @varr fn-symb))))
(defn ns-wrap-try-catch!
"Re-intern all functions in NAMESPACE as ones that wrap the originals with a `try-catch`.
Defaults to the current namespace. You may optionally exclude a set of symbols using the kwarg `:exclude`.
(ns-wrap-try-catch!)
(ns-wrap-try-catch! 'metabase.driver)
(ns-wrap-try-catch! 'metabase.driver :exclude 'query-complete)
Intended for debugging."
{:arglists '([namespace? :exclude & excluded-symbs])}
[& args]
(let [[nmspc args] (optional #(try-apply the-ns [%]) args *ns*)
excluded (when (= (first args) :exclude)
(set (rest args)))]
(doseq [[symb varr] (ns-interns nmspc)]
(when (fn? @varr)
(when-not (contains? excluded symb)
(wrap-try-catch! (symbol (str (ns-name nmspc) \/ symb))))))))
(defn deref-with-timeout (defn deref-with-timeout
"Call `deref` on a FUTURE and throw an exception if it takes more than TIMEOUT-MS." "Call `deref` on a FUTURE and throw an exception if it takes more than TIMEOUT-MS."
[futur timeout-ms] [futur timeout-ms]
......
test/metabase/driver/query_processor/expand_resolve_test.clj 0 → 100644
+ 296
− 0
View file @ e30c6d1d
(ns metabase.driver.query-processor.expand-resolve-test
"Tests query expansion/resolution"
(:require [expectations :refer :all]
[metabase.driver.query-processor.expand :as expand]
[metabase.driver.query-processor.resolve :as resolve]
[metabase.test.data :refer :all]
[metabase.util :as u]))
;; this is here because expectations has issues comparing and object w/ a map and most of the output
;; below has objects for the various place holders in the expanded/resolved query
(defn- obj->map [o]
(cond
(sequential? o) (vec (for [v o]
(obj->map v)))
(set? o) (set (for [v o]
(obj->map v)))
(map? o) (into {} (for [[k v] o]
{k (obj->map v)}))
:else o))
;; basic rows query w/ filter
(expect
[;; expanded form
{:database (id)
:type :query
:query {:source-table (id :venues)
:filter {:filter-type :>
:field {:field-id (id :venues :price)
:fk-field-id nil
:datetime-unit nil}
:value {:field-placeholder {:field-id (id :venues :price)
:fk-field-id nil
:datetime-unit nil}
:value 1}}}}
;; resolved form
{:database {:name "test-data"
:details {:short-lived? nil
:db "mem:test-data;USER=GUEST;PASSWORD=guest"}
:id (id)
:engine :h2}
:type :query
:query {:source-table {:schema "PUBLIC"
:name "VENUES"
:id (id :venues)}
:filter {:filter-type :>
:field {:field-id (id :venues :price)
:field-name "PRICE"
:field-display-name "Price"
:base-type :IntegerField
:special-type :category
:visibility-type :normal
:table-id (id :venues)
:schema-name "PUBLIC"
:table-name "VENUES"
:position nil
:description nil
:parent-id nil
:parent nil}
:value {:value 1
:field {:field-id (id :venues :price)
:field-name "PRICE"
:field-display-name "Price"
:base-type :IntegerField
:special-type :category
:visibility-type :normal
:table-id (id :venues)
:schema-name "PUBLIC"
:table-name "VENUES"
:position nil
:description nil
:parent-id nil
:parent nil}}}
:join-tables nil}
:fk-field-ids #{}
:table-ids #{(id :venues)}}]
(let [expanded-form (expand/expand {:database (id)
:type :query
:query {:source-table (id :venues)
:aggregation ["rows"]
:filter ["AND" [">" ["field-id" (id :venues :price)] 1]]}})]
(mapv obj->map [expanded-form
(resolve/resolve expanded-form)])))
;; basic rows query w/ FK filter
(expect
[;; expanded form
{:database (id)
:type :query
:query {:source-table (id :venues)
:filter {:filter-type :=
:field {:field-id (id :categories :name)
:fk-field-id (id :venues :category_id)
:datetime-unit nil}
:value {:field-placeholder {:field-id (id :categories :name)
:fk-field-id (id :venues :category_id)
:datetime-unit nil}
:value "abc"}}}}
;; resolved form
{:database {:name "test-data"
:details {:short-lived? nil
:db "mem:test-data;USER=GUEST;PASSWORD=guest"}
:id (id)
:engine :h2}
:type :query
:query {:source-table {:schema "PUBLIC"
:name "VENUES"
:id (id :venues)}
:filter {:filter-type :=
:field {:field-id (id :categories :name)
:field-name "NAME"
:field-display-name "Name"
:base-type :TextField
:special-type :name
:visibility-type :normal
:table-id (id :categories)
:schema-name "PUBLIC"
:table-name "CATEGORIES"
:position nil
:description nil
:parent-id nil
:parent nil}
:value {:value "abc"
:field {:field-id (id :categories :name)
:field-name "NAME"
:field-display-name "Name"
:base-type :TextField
:special-type :name
:visibility-type :normal
:table-id (id :categories)
:schema-name "PUBLIC"
:table-name "CATEGORIES"
:position nil
:description nil
:parent-id nil
:parent nil}}}
:join-tables [{:source-field {:field-id (id :venues :category_id)
:field-name "CATEGORY_ID"}
:pk-field {:field-id (id :categories :id)
:field-name "ID"}
:table-id (id :categories)
:table-name "CATEGORIES"
:schema "PUBLIC"}]}
:fk-field-ids #{(id :venues :category_id)}
:table-ids #{(id :categories)}}]
(let [expanded-form (expand/expand {:database (id)
:type :query
:query {:source-table (id :venues)
:aggregation ["rows"]
:filter ["AND" ["=" ["fk->" (id :venues :category_id) (id :categories :name)] "abc"]]}})]
(mapv obj->map [expanded-form
(resolve/resolve expanded-form)])))
;; basic rows query w/ FK filter on datetime
(expect
[;; expanded form
{:database (id)
:type :query
:query {:source-table (id :checkins)
:filter {:filter-type :>
:field {:field-id (id :users :last_login)
:fk-field-id (id :checkins :user_id)
:datetime-unit :year}
:value {:field-placeholder {:field-id (id :users :last_login)
:fk-field-id (id :checkins :user_id)
:datetime-unit :year}
:value "1980-01-01"}}}}
;; resolved form
{:database {:name "test-data"
:details {:short-lived? nil
:db "mem:test-data;USER=GUEST;PASSWORD=guest"}
:id (id)
:engine :h2}
:type :query
:query {:source-table {:schema "PUBLIC"
:name "CHECKINS"
:id (id :checkins)}
:filter {:filter-type :>
:field {:field {:field-id (id :users :last_login)
:field-name "LAST_LOGIN"
:field-display-name "Last Login"
:base-type :DateTimeField
:special-type nil
:visibility-type :normal
:table-id (id :users)
:schema-name "PUBLIC"
:table-name "USERS"
:position nil
:description nil
:parent-id nil
:parent nil}
:unit :year}
:value {:value (u/->Timestamp "1980-01-01")
:field {:field {:field-id (id :users :last_login)
:field-name "LAST_LOGIN"
:field-display-name "Last Login"
:base-type :DateTimeField
:special-type nil
:visibility-type :normal
:table-id (id :users)
:schema-name "PUBLIC"
:table-name "USERS"
:position nil
:description nil
:parent-id nil
:parent nil}
:unit :year}}}
:join-tables [{:source-field {:field-id (id :checkins :user_id)
:field-name "USER_ID"}
:pk-field {:field-id (id :users :id)
:field-name "ID"}
:table-id (id :users)
:table-name "USERS"
:schema "PUBLIC"}]}
:fk-field-ids #{(id :checkins :user_id)}
:table-ids #{(id :users)}}]
(let [expanded-form (expand/expand {:database (id)
:type :query
:query {:source-table (id :checkins)
:aggregation ["rows"]
:filter ["AND" [">" ["datetime-field" ["fk->" (id :checkins :user_id) (id :users :last_login)] "year"] "1980-01-01"]]}})]
(mapv obj->map [expanded-form
(resolve/resolve expanded-form)])))
;; sum aggregation w/ datetime breakout
(expect
[;; expanded form
{:database (id)
:type :query
:query {:source-table (id :checkins)
:aggregation {:aggregation-type :sum
:field {:field-id (id :venues :price)
:fk-field-id (id :checkins :venue_id)
:datetime-unit nil}}
:breakout [{:field-id (id :checkins :date)
:fk-field-id nil
:datetime-unit :day-of-week}]}}
;; resolved form
{:database {:name "test-data"
:details {:short-lived? nil
:db "mem:test-data;USER=GUEST;PASSWORD=guest"}
:id (id)
:engine :h2}
:type :query
:query {:source-table {:schema "PUBLIC"
:name "CHECKINS"
:id (id :checkins)}
:aggregation {:aggregation-type :sum
:field {:description nil
:base-type :IntegerField
:parent nil
:table-id (id :venues)
:special-type :category
:field-name "PRICE"
:field-display-name "Price"
:parent-id nil
:visibility-type :normal
:position nil
:field-id (id :venues :price)
:table-name "VENUES"
:schema-name "PUBLIC"}}
:breakout [{:field {:description nil
:base-type :DateField
:parent nil
:table-id (id :checkins)
:special-type nil
:field-name "DATE"
:field-display-name "Date"
:parent-id nil
:visibility-type :normal
:position nil
:field-id (id :checkins :date)
:table-name "CHECKINS"
:schema-name "PUBLIC"}
:unit :day-of-week}]
:join-tables [{:source-field {:field-id (id :checkins :venue_id)
:field-name "VENUE_ID"}
:pk-field {:field-id (id :venues :id)
:field-name "ID"}
:table-id (id :venues)
:table-name "VENUES"
:schema "PUBLIC"}]}
:fk-field-ids #{(id :checkins :venue_id)}
:table-ids #{(id :venues) (id :checkins)}}]
(let [expanded-form (expand/expand {:database (id)
:type :query
:query {:source-table (id :checkins)
:aggregation ["sum" ["fk->" (id :checkins :venue_id) (id :venues :price)]]
:breakout [["datetime-field" (id :checkins :date) "day-of-week"]]
:filter []}})]
(mapv obj->map [expanded-form
(resolve/resolve expanded-form)])))
test/metabase/driver/qp_middleware_test.clj → test/metabase/driver/query_processor/qp_middleware_test.clj
+ 55
− 2
View file @ e30c6d1d
(ns metabase.driver.qp-middleware-test (ns metabase.driver.query_processor.qp-middleware-test
(:require [expectations :refer :all] (:require [expectations :refer :all]
[clj-time.coerce :as tc] [clj-time.coerce :as tc]
[metabase.driver :as driver] [metabase.driver :as driver]
...@@ -6,7 +6,8 @@ ...@@ -6,7 +6,8 @@
[metabase.models.setting :as setting] [metabase.models.setting :as setting]
[metabase.test.util :refer [resolve-private-fns]])) [metabase.test.util :refer [resolve-private-fns]]))
(resolve-private-fns metabase.driver.query-processor post-format-rows pre-add-settings) (resolve-private-fns metabase.driver.query-processor
wrap-catch-exceptions post-add-row-count-and-status post-format-rows pre-add-settings)
(defrecord TestDriver [] (defrecord TestDriver []
...@@ -18,6 +19,23 @@ ...@@ -18,6 +19,23 @@
{:features (constantly #{:set-timezone})}) {:features (constantly #{:set-timezone})})
;; wrap-catch-exceptions
(expect
{}
((wrap-catch-exceptions identity) {}))
(expect
{:status :failed
:class java.lang.Exception
:error "Something went wrong"
:stacktrace true
:query {}
:expanded-query nil}
(-> ((wrap-catch-exceptions (fn [_] (throw (Exception. "Something went wrong")))) {})
(update :stacktrace boolean)))
;; pre-add-settings ;; pre-add-settings
(expect (expect
...@@ -40,6 +58,32 @@ ...@@ -40,6 +58,32 @@
(dissoc response3 :driver)])))) (dissoc response3 :driver)]))))
;; TODO: pre-expand-macros
;; TODO: pre-expand-resolve
;; post-add-row-count-and-status
(expect
{:row_count 5
:status :completed
:data {:rows [[1] [1] [1] [1] [1]]
:rows_truncated 5}}
;; NOTE: the default behavior is to treat the query as :rows type aggregation and use :max-results-bare-rows
((post-add-row-count-and-status (constantly {:rows [[1] [1] [1] [1] [1]]}))
{:constraints {:max-results 10
:max-results-bare-rows 5}}))
(expect
{:row_count 5
:status :completed
:data {:rows [[1] [1] [1] [1] [1]]}}
;; when we aren't a :rows query the then we use :max-results for our limit
((post-add-row-count-and-status (constantly {:rows [[1] [1] [1] [1] [1]]}))
{:query {:aggregation {:aggregation-type :count}}
:constraints {:max-results 10
:max-results-bare-rows 5}}))
;; post-format-rows ;; post-format-rows
(expect (expect
...@@ -57,3 +101,12 @@ ...@@ -57,3 +101,12 @@
((post-format-rows (constantly {:rows [[(tc/to-sql-time 1303121567232)] ((post-format-rows (constantly {:rows [[(tc/to-sql-time 1303121567232)]
[(tc/to-sql-date "2011-04-18")] ; joda-time assumes this is UTC time when parsing it [(tc/to-sql-date "2011-04-18")] ; joda-time assumes this is UTC time when parsing it
[(tc/to-date 1303121567232)]]})) {:settings {:report-timezone "Asia/Tokyo"}})) [(tc/to-date 1303121567232)]]})) {:settings {:report-timezone "Asia/Tokyo"}}))
;; TODO: pre-add-implicit-fields
;; TODO: pre-add-implicit-breakout-order-by
;; TODO: cumulative-sum
;; TODO: limit
;; TODO: post-annotate
;; TODO: pre-log-query
;; TODO: wrap-guard-multiple-calls
test/metabase/util_test.clj
+ 0
− 41
View file @ e30c6d1d
...@@ -46,17 +46,6 @@ ...@@ -46,17 +46,6 @@
(expect #inst "2015-11-01" (date-trunc :month friday-the-13th)) (expect #inst "2015-11-01" (date-trunc :month friday-the-13th))
(expect #inst "2015-10-01" (date-trunc :quarter friday-the-13th)) (expect #inst "2015-10-01" (date-trunc :quarter friday-the-13th))
;;; ## tests for ASSOC<>
(expect
{:a 100
:b 200
:c 300}
(assoc<> {}
:a 100
:b (+ 100 (:a <>))
:c (+ 100 (:b <>))))
;;; ## tests for HOST-UP? ;;; ## tests for HOST-UP?
(expect true (expect true
...@@ -71,36 +60,6 @@ ...@@ -71,36 +60,6 @@
(host-port-up? "nosuchhost" 8005)) (host-port-up? "nosuchhost" 8005))
;; ## tests for `(format-num)`
;; basic whole number case
(expect "1" (format-num 1))
(expect "1" (format-num (float 1)))
(expect "1" (format-num (double 1)))
(expect "1" (format-num (bigdec 1)))
(expect "1" (format-num (long 1)))
;; make sure we correctly format down to 2 decimal places
;; note that we are expecting a round DOWN in this case
(expect "1.23" (format-num (float 1.23444)))
(expect "1.23" (format-num (double 1.23444)))
(expect "1.23" (format-num (bigdec 1.23444)))
;; test that we always force precision of 2 on decimal places
(expect "1.20" (format-num (float 1.2)))
(expect "1.20" (format-num (double 1.2)))
(expect "1.20" (format-num (bigdec 1.2)))
;; we can take big numbers and add in commas
(expect "1,234" (format-num 1234))
(expect "1,234" (format-num (float 1234)))
(expect "1,234" (format-num (double 1234)))
(expect "1,234" (format-num (bigdec 1234)))
(expect "1,234" (format-num (long 1234)))
;; we can handle numbers with both commas and decimal places
;; note that we expect a basic round UP here
(expect "1,234.57" (format-num (float 1234.5678)))
(expect "1,234.57" (format-num (double 1234.5678)))
(expect "1,234.57" (format-num (bigdec 1234.5678)))
;;; ## tests for IS-URL? ;;; ## tests for IS-URL?
(expect true (is-url? "http://google.com")) (expect true (is-url? "http://google.com"))
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment