Skip to content
Snippets Groups Projects
Commit d5c87c05 authored by Cam Saul's avatar Cam Saul
Browse files

rewritten Q macro; nested filter support + vararg =/!=

parent e778d5eb
Branches
Tags
No related merge requests found
Expand all Hide whitespace changes
Inline Side-by-side
src/metabase/driver/generic_sql/query_processor.clj
+ 8
− 6
View file @ d5c87c05
......@@ -192,12 +192,14 @@
:= {field ['= value]}
:!= {field ['not= value]}))))
(defmethod apply-form :filter [[_ {:keys [compound-type subclauses]}]]
(let [[first-subclause :as subclauses] (map filter-subclause->predicate subclauses)]
`(where ~(case compound-type
:and `(~'and ~@subclauses)
:or `(~'or ~@subclauses)
:simple first-subclause))))
(defn- filter-clause->predicate [{:keys [compound-type subclauses], :as clause}]
(case compound-type
:and `(~'and ~@(map filter-clause->predicate subclauses))
:or `(~'or ~@(map filter-clause->predicate subclauses))
nil (filter-subclause->predicate clause)))
(defmethod apply-form :filter [[_ clause]]
`(where ~(filter-clause->predicate clause)))
(defmethod apply-form :join-tables [[_ join-tables]]
......
src/metabase/driver/query_processor/expand.clj
+ 20
− 5
View file @ d5c87c05
......@@ -423,11 +423,27 @@
:min-val (ph field-id min)
:max-val (ph field-id max)})
[(filter-type :guard (partial contains? #{"=" "!=" "<" ">" "<=" ">="})) (field-id :guard Field?) (val :guard (complement nil?))]
[(filter-type :guard (partial contains? #{"!=" "=" "<" ">" "<=" ">="})) (field-id :guard Field?) (val :guard (complement nil?))]
(map->Filter:Field+Value {:filter-type (keyword filter-type)
:field (ph field-id)
:value (ph field-id val)})
;; = with more than one value -- Convert to OR and series of = clauses
["=" (field-id :guard Field?) & (values :guard #(and (seq %) (every? (complement nil?) %)))]
(map->Filter {:compound-type :or
:subclauses (vec (for [value values]
(map->Filter:Field+Value {:filter-type :=
:field (ph field-id)
:value (ph field-id value)})))})
;; != with more than one value -- Convert to AND and series of != clauses
["!=" (field-id :guard Field?) & (values :guard #(and (seq %) (every? (complement nil?) %)))]
(map->Filter {:compound-type :and
:subclauses (vec (for [value values]
(map->Filter:Field+Value {:filter-type :!=
:field (ph field-id)
:value (ph field-id value)})))})
[(filter-type :guard (partial contains? #{"STARTS_WITH" "CONTAINS" "ENDS_WITH"})) (field-id :guard Field?) (val :guard string?)]
(map->Filter:Field+Value {:filter-type (case filter-type
"STARTS_WITH" :starts-with
......@@ -444,11 +460,10 @@
(defparser parse-filter
["AND" & subclauses] (map->Filter {:compound-type :and
:subclauses (mapv parse-filter-subclause subclauses)})
:subclauses (mapv parse-filter subclauses)})
["OR" & subclauses] (map->Filter {:compound-type :or
:subclauses (mapv parse-filter-subclause subclauses)})
subclause (map->Filter {:compound-type :simple
:subclauses [(parse-filter-subclause subclause)]}))
:subclauses (mapv parse-filter subclauses)})
subclause (parse-filter-subclause subclause))
;; ## -------------------- Order-By --------------------
......
test/metabase/driver/postgres_test.clj
+ 4
− 4
View file @ d5c87c05
......@@ -3,7 +3,7 @@
[metabase.driver.generic-sql.interface :as i]
[metabase.driver.postgres :refer :all]
[metabase.test.data.interface :refer [def-database-definition]]
[metabase.test.util.mql :refer [Q]]))
[metabase.test.util.q :refer [Q]]))
;; # Check that database->connection details still works whether we're dealing with new-style or legacy details
;; ## new-style
......@@ -67,14 +67,14 @@
[3 #uuid "da1d6ecc-e775-4008-b366-c38e7a2e8433"]
[4 #uuid "7a5ce4a2-0958-46e7-9685-1a4eaa3bd08a"]
[5 #uuid "84ed434e-80b4-41cf-9c88-e334427104ae"]]}
(-> (Q run against metabase.driver.postgres-test/with-uuid using postgres
(-> (Q dataset metabase.driver.postgres-test/with-uuid use postgres
return :data
aggregate rows of users)
(update :cols (partial mapv #(dissoc % :id :table_id)))))
;; Check that we can filter by a UUID Field
(expect [[2 #uuid "4652b2e7-d940-4d55-a971-7e484566663e"]]
(Q run against metabase.driver.postgres-test/with-uuid using postgres
return :data :rows
(Q dataset metabase.driver.postgres-test/with-uuid use postgres
return rows
aggregate rows of users
filter = user_id "4652b2e7-d940-4d55-a971-7e484566663e"))
test/metabase/driver/query_processor_test.clj
+ 367
− 446
View file @ d5c87c05
This diff is collapsed.
test/metabase/test/util/mql.clj deleted 100644 → 0
+ 0
− 210
View file @ e778d5eb
(ns metabase.test.util.mql
"DSL for writing metabase QL queries."
(:require [clojure.core.match :refer [match]]
[clojure.tools.macro :refer :all]
[clojure.walk :refer [macroexpand-all]]
[metabase.driver :as driver]
[metabase.test.data :as data]
[metabase.test.data.datasets :as datasets]
[metabase.util :as u]))
(defn- partition-tokens [keywords tokens]
(->> (loop [all [], current-split nil, [token & more] tokens]
(cond
(and (nil? token)
(not (seq more))) (conj all current-split)
(contains? keywords token) (recur (or (when (seq current-split)
(conj all current-split))
all)
[token]
more)
:else (recur all
(conj current-split token)
more)))
(map seq)
(filter (complement nil?))))
(def ^:private ^:const outer-q-tokens '#{against with run return using})
(def ^:private ^:const inner-q-tokens '#{ag aggregate breakout fields filter lim limit order page of tbl})
(defmacro Q:temp-get [& args]
`(:id (data/-temp-get ~'db ~@(map name args))))
(defn Q:resolve-dataset [^clojure.lang.Symbol dataset]
;; Try to resolve symbol as-is, otherwise try to resolve in the metabase.test.data.dataset-definitions namespace
(var-get (or (resolve dataset)
(do (require 'metabase.test.data.dataset-definitions)
(ns-resolve 'metabase.test.data.dataset-definitions dataset))
(throw (Exception. (format "Don't know how to find dataset '%s'." dataset))))))
(defmacro Q:with-temp-db [dataset body]
`(data/with-temp-db [~'db (data/dataset-loader) (Q:resolve-dataset '~dataset)]
(symbol-macrolet [~'db-id (:id ~'db)]
(macrolet [(~'id [& args#] `(Q:temp-get ~@args#))]
~(macroexpand-all body)))))
(defmacro Q:against [query arg]
`(Q:with-temp-db ~arg
~query))
(defmacro Q:using [query arg]
`(datasets/with-dataset ~(keyword arg)
~query))
(defmacro Q:with [query arg & [arg2 :as more]]
(case (keyword arg)
:db `(Q:against ~query ~arg2)
:dataset `(Q:using ~query ~arg2)
:datasets `(do ~@(for [dataset# more]
`(datasets/with-dataset ~(keyword dataset#)
~query)))))
(defmacro Q:return [q & args]
`(->> ~q ~@args))
(defmacro Q:expand-outer [token form]
(macroexpand-all `(symbol-macrolet [~'against Q:against
~'return Q:return
~'run driver/process-query
~'using Q:using
~'with Q:with]
(-> ~form ~token))))
(defmacro Q:expand-outer* [[token & tokens] form]
(if-not token form
`(Q:expand-outer* ~tokens (Q:expand-outer ~token ~form))))
(defmacro Q:expand-inner [& forms]
{:database 'db-id
:type :query
:query `(Q:expand-clauses {} ~@forms)})
(defmacro Q:wrap-fallback-captures [form]
`(symbol-macrolet [~'db-id (data/db-id)
~'id data/id]
~(macroexpand-all form)))
(defmacro Q:field [f]
(or (when (symbol? f)
(let [f (name f)]
(u/cond-let
[[_ from to] (re-matches #"^(.+)->(.+)$" f)] ["fk->" `(Q:field ~(symbol from)) `(Q:field ~(symbol to))]
[[_ f sub] (re-matches #"^(.+)\.\.\.(.+)$" f)] `(~@(macroexpand-1 `(Q:field ~(symbol f))) ~(keyword sub))
[[_ ag-field-index] (re-matches #"^ag\.(\d+)$" f)] ["aggregation" (Integer/parseInt ag-field-index)]
[[_ table field] (re-matches #"^(?:([^\.]+)\.)?([^\.]+)$" f)] `(~'id ~(if table (keyword table)
'table)
~(keyword field)))))
f))
(defmacro Q [& tokens]
(let [[outer-tokens inner-tokens] (split-with (complement (partial contains? inner-q-tokens)) tokens)
outer-tokens (partition-tokens outer-q-tokens outer-tokens)
inner-tokens (partition-tokens inner-q-tokens inner-tokens)
query (macroexpand-all `(Q:expand-inner ~@inner-tokens))
table (second (:source_table (:query query)))]
(assert table "No table specified. Did you include a `tbl`/`of` clause?")
`(Q:wrap-fallback-captures (Q:expand-outer* ~outer-tokens
(symbol-macrolet [~'table ~table
~'fl Q:field]
~(macroexpand-all query))))))
(defmacro Q:expand-clauses [acc & [[clause & args] & more]]
(if-not clause acc
`(Q:expand-clauses ~(macroexpand-all `(~(symbol (format "metabase.test.util.mql/Q:%s" clause)) ~acc ~@args)) ~@more)))
;; ## ag
(defmacro Q:ag [query & tokens]
(assoc query :aggregation (match (vec tokens)
['rows] ["rows"]
['count] ["count"]
['count id] ["count" `(~'fl ~id)]
['avg id] ["avg" `(~'fl ~id)]
['distinct id] ["distinct" `(~'fl ~id)]
['stddev id] ["stddev" `(~'fl ~id)]
['sum id] ["sum" `(~'fl ~id)]
['cum-sum id] ["cum_sum" `(~'fl ~id)])))
(defmacro Q:aggregate [& args]
`(Q:ag ~@args))
;; ## breakout
(defmacro Q:breakout [query & fields]
(assoc query :breakout (vec (for [field fields]
`(~'fl ~field)))))
;; ## fields
(defmacro Q:fields [query & fields]
(assoc query :fields (vec (for [field fields]
`(~'fl ~field)))))
;; ## filter
(def ^:private ^:const filter-tokens
'#{inside not-null is-null between = != < > <= >=})
(defmacro Q:filter [query & tokens]
(assoc query :filter `(Q:filter* ~tokens)))
(defmacro Q:filter* [[subclause & [arg arg2 :as args]]]
(case (keyword subclause)
:and `["AND" ~@(for [cl (partition-tokens filter-tokens args)]
`(Q:filter* ~cl))]
:or `["OR" ~@(for [cl (partition-tokens filter-tokens args)]
`(Q:filter* ~cl))]
:inside (let [{:keys [lat lon]} arg]
["INSIDE" `(~'fl ~(:field lat)) `(~'fl ~(:field lon)) (:max lat) (:min lon) (:min lat) (:max lon)])
:not-null ["NOT_NULL" `(~'fl ~arg)]
:is-null ["IS_NULL" `(~'fl ~arg)]
:between (let [[id min max] args]
["BETWEEN" `(~'fl ~id) ~min ~max])
:starts-with ["STARTS_WITH" `(~'fl ~arg) arg2]
:ends-with ["ENDS_WITH" `(~'fl ~arg) arg2]
:contains ["CONTAINS" `(~'fl ~arg) arg2]
:= ["=" `(~'fl ~arg) arg2]
:!= ["!=" `(~'fl ~arg) arg2]
:< ["<" `(~'fl ~arg) arg2]
:> [">" `(~'fl ~arg) arg2]
:<= ["<=" `(~'fl ~arg) arg2]
:>= [">=" `(~'fl ~arg) arg2]))
;; ## lim
(defmacro Q:lim [query lim]
(assoc query :limit lim))
(defmacro Q:limit [& args]
`(Q:lim ~@args))
;; ## order
(defmacro Q:order [query & fields]
(assoc query :order_by (vec (for [field fields]
`(Q:order* ~field)))))
(defmacro Q:order* [field-symb]
(let [[_ field +-] (re-matches #"^(.+[^\-+])([\-+])?$" (name field-symb))]
(assert field (format "Invalid field passed to order: '%s'" field-symb))
[`(~'fl ~(symbol field)) (case (keyword (or +- '+))
:+ "ascending"
:- "descending")]))
;; ## page
(defmacro Q:page [query page items]
(assoc query :page {:page page
:items items}))
;; ## tbl
(defmacro Q:tbl [query table]
(assoc query :source_table `(~'id ~(keyword table))))
(defmacro Q:of [query table]
`(Q:tbl ~query ~table))
test/metabase/test/util/q.clj 0 → 100644
+ 281
− 0
View file @ d5c87c05
(ns metabase.test.util.q
(:refer-clojure :exclude [or and filter use = != < > <= >=])
(:require [clojure.core :as core]
[clojure.core.match :refer [match]]
[metabase.driver :as driver]
[metabase.test.data :as data]
(metabase.test.data [datasets :as datasets]
dataset-definitions)
[metabase.util :as u]))
;;; # HELPER FNs
;;; ## TOKEN SPLITTING
(def ^:private ^:const top-level-tokens
'#{use of dataset return aggregate breakout fields filter limit order page})
(defn- qualify-token [token]
(symbol (str "metabase.test.util.q/" token)))
(defn- qualify-form [[f & args]]
`(~(qualify-token f) ~@args))
(defn- split-with-tokens [tokens args]
(loop [acc [], current-group [], [arg & more] args]
(cond
(nil? arg) (->> (conj acc (apply list current-group))
(core/filter seq)
(map qualify-form))
(contains? tokens arg) (recur (conj acc (apply list current-group)) [arg] more)
:else (recur acc (conj current-group arg) more))))
;;; ## ID LOOKUP
(def ^:dynamic *db* nil)
(def ^:dynamic *table-name* nil)
(defn db-id []
{:post [(integer? %)]}
(core/or (:id *db*)
(data/db-id)))
(defn id [& args]
{:pre [(every? keyword? args)]
:post [(core/or (integer? %)
(println (str "Couldn't find ID of: " args)))]}
(if *db*
(:id (apply data/-temp-get *db* (map name args)))
(apply data/id args)))
(defmacro field [f]
{:pre [(symbol? f)]}
(core/or
(let [f (name f)]
(u/cond-let
;; x->y <-> ["fk->" x y]
[[_ from to] (re-matches #"^(.+)->(.+)$" f)]
["fk->" `(field ~(symbol from)) `(field ~(symbol to))]
;; x...y <-> ?
[[_ f sub] (re-matches #"^(.+)\.\.\.(.+)$" f)]
`(~@(macroexpand-1 `(field ~(symbol f))) ~(keyword sub))
;; ag.0 <-> ["aggregation" 0]
[[_ ag-field-index] (re-matches #"^ag\.(\d+)$" f)]
["aggregation" (Integer/parseInt ag-field-index)]
;; table.field <-> (id table field)
[[_ table field] (re-matches #"^([^\.]+)\.([^\.]+)$" f)]
`(id ~(keyword table) ~(keyword field))))
;; fallback : (id *table-name* field)
`(id *table-name* ~(keyword f))))
(defn resolve-dataset [dataset]
(var-get (core/or (resolve dataset)
(ns-resolve 'metabase.test.data.dataset-definitions dataset)
(throw (Exception. (format "Don't know how to find dataset '%s'." dataset))))))
;;; # DSL KEYWORD MACROS
;;; ## USE
(defmacro use [query db]
(assoc-in query [:context :driver] (keyword db)))
;;; ## OF
(defmacro of [query table-name]
(-> query
(assoc-in [:query :source_table] `(id ~(keyword table-name)))
(assoc-in [:context :table-name] (keyword table-name))))
;;; ## DATASET
(defmacro dataset [query dataset-name]
(assoc-in query [:context :dataset] `'~dataset-name))
;;; ## RETURN
(defmacro return [query & args]
(assoc-in query [:context :return] (vec (mapcat (fn [arg]
(cond
(core/= arg 'rows) [:data :rows]
(core/= arg 'first-row) [:data :rows first]
:else [arg]))
args))))
;;; ## AGGREGATE
(defmacro aggregate [query & args]
(assoc-in query [:query :aggregation] (match (vec args)
['rows] ["rows"]
['count] ["count"]
['count id] ["count" `(field ~id)]
['avg id] ["avg" `(field ~id)]
['distinct id] ["distinct" `(field ~id)]
['stddev id] ["stddev" `(field ~id)]
['sum id] ["sum" `(field ~id)]
['cum-sum id] ["cum_sum" `(field ~id)])))
;;; ## BREAKOUT
(defmacro breakout [query & fields]
(assoc-in query [:query :breakout] (vec (for [field fields]
`(field ~field)))))
;;; ## FIELDS
(defmacro fields [query & fields]
(assoc-in query [:query :fields] (vec (for [field fields]
`(field ~field)))))
;;; ## FILTER
(def ^:const ^:private filter-clause-tokens
'#{inside not-null is-null between starts-with ends-with contains = != < > <= >=})
(defmacro and [& clauses]
`["AND" ~@clauses])
(defmacro or [& clauses]
`["OR" ~@clauses])
(defmacro inside [{:keys [lat lon]}]
`["INSIDE" (field ~(:field lat)) (field ~(:field lon)) ~(:max lat) ~(:min lon) ~(:min lat) ~(:max lon)])
(defmacro not-null [field]
`["NOT_NULL" (field ~field)])
(defmacro is-null [field]
`["IS_NULL" (field ~field)])
(defmacro between [field min max]
`["BETWEEN" (field ~field) ~min ~max])
(defmacro starts-with [field arg]
`["STARTS_WITH" (field ~field) ~arg])
(defmacro ends-with [field arg]
`["ENDS_WITH" (field ~field) ~arg])
(defmacro contains [field arg]
`["CONTAINS" (field ~field) ~arg])
(defmacro = [field & args]
`["=" (field ~field) ~@args])
(defmacro != [field & args]
`["!=" (field ~field) ~@args])
(defmacro < [field arg]
`["<" (field ~field) ~arg])
(defmacro <= [field arg]
`["<=" (field ~field) ~arg])
(defmacro > [field arg]
`[">" (field ~field) ~arg])
(defmacro >= [field arg]
`[">=" (field ~field) ~arg])
(defn- filter-split [tokens]
(->> (loop [clauses [], current-clause [], [token & more] tokens]
(cond
(nil? token) (conj clauses (apply list current-clause))
(core/= token 'and) (conj clauses (apply list current-clause) `(and ~@(filter-split more)))
(core/= token 'or) (conj clauses (apply list current-clause) `(or ~@(filter-split more)))
(contains? filter-clause-tokens token) (recur (conj clauses (apply list current-clause))
[(qualify-token token)]
more)
:else (recur clauses
(conj current-clause token)
more)))
(core/filter seq)))
(defmacro filter* [& args]
(first (filter-split args)))
(defmacro filter [query & args]
(assoc-in query [:query :filter] `(filter* ~@args)))
;;; ## LIMIT
(defmacro limit [query limit]
{:pre [(integer? limit)]}
(assoc-in query [:query :limit] limit))
;;; ## ORDER
(defmacro order* [field-symb]
(let [[_ field +-] (re-matches #"^(.+[^\-+])([\-+])?$" (name field-symb))]
(assert field (format "Invalid field passed to order: '%s'" field-symb))
[`(field ~(symbol field)) (case (keyword (core/or +- '+))
:+ "ascending"
:- "descending")]))
(defmacro order [query & fields]
(assoc-in query [:query :order_by] (vec (for [field fields]
`(order* ~field)))))
;;; ## PAGE
(defmacro page [query page items-symb items]
(assert (and (integer? page)
(core/= items-symb 'items)
(integer? items))
"page clause should be of the form page <page-num> items <items-per-page>")
(assoc-in query [:query :page] {:page page
:items items}))
;;; # TOP-LEVEL MACRO IMPL
(defmacro with-temp-db [dataset query]
(if-not dataset
query
`(data/with-temp-db [db# (data/dataset-loader) (resolve-dataset ~dataset)]
(binding [*db* db#]
~query))))
(defmacro with-driver [driver query]
(if-not driver
query
`(datasets/with-dataset ~driver
~query)))
(defmacro Q** [{:keys [driver dataset return table-name]} query]
(assert table-name
"Table name not specified in query, did you include an 'of' clause?")
`(->> (with-driver ~driver
(binding [*table-name* ~table-name]
(with-temp-db ~dataset
(driver/process-query ~query))))
~@return))
(defmacro Q* [q & [form & more]]
(if-not form
`(Q** ~(:context q) ~(dissoc q :context))
`(Q* ~(macroexpand `(-> ~q ~form)) ~@more)))
(defmacro Q [& args]
`(Q* {:database (db-id)
:type :query
:query {}
:context {:driver nil
:dataset nil}}
~@(split-with-tokens top-level-tokens args)))
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment