Skip to content
Snippets Groups Projects
Commit f2bf8fd7 authored by Cam Saül's avatar Cam Saül
Browse files

Don't need to eval in SQL QP

parent 3dd11581
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
src/metabase/driver/generic_sql/query_processor.clj
+ 120
− 154
View file @ f2bf8fd7
......@@ -3,9 +3,10 @@
(:require [clojure.core.match :refer [match]]
[clojure.tools.logging :as log]
[clojure.string :as s]
[clojure.walk :as walk]
[korma.core :refer :all, :exclude [update]]
[korma.sql.utils :as utils]
(korma [core :as k]
[db :as kdb])
(korma.sql [fns :as kfns]
[utils :as utils])
[metabase.config :as config]
[metabase.driver :as driver]
[metabase.driver.query-processor :as qp]
......@@ -21,73 +22,9 @@
RelativeDateTimeValue
Value)))
(declare apply-form
log-korma-form)
(def ^:private ^:dynamic *query* nil)
;; # INTERFACE
(def ^:dynamic ^:private *query* nil)
(defn process-structured
"Convert QUERY into a korma `select` form, execute it, and annotate the results."
[{{:keys [source-table]} :query, database :database, :as query}]
(binding [*query* query]
(try
;; Process the expanded query and generate a korma form
(let [korma-select-form `(select ~'entity ~@(->> (map apply-form (:query query))
(filter identity)
(mapcat #(if (vector? %) % [%]))))
set-timezone-sql (when-let [timezone (driver/report-timezone)]
(when (seq timezone)
(let [{:keys [features timezone->set-timezone-sql]} (:driver *query*)]
(when (contains? features :set-timezone)
`(exec-raw ~(timezone->set-timezone-sql timezone))))))
korma-form `(let [~'entity (korma-entity ~database ~source-table)]
~(if set-timezone-sql `(korma.db/with-db (:db ~'entity)
(korma.db/transaction
~set-timezone-sql
~korma-select-form))
korma-select-form))]
;; Log generated korma form
(when (config/config-bool :mb-db-logging)
(log-korma-form korma-form))
(eval korma-form))
(catch java.sql.SQLException e
(let [^String message (or (->> (.getMessage e) ; error message comes back like "Error message ... [status-code]" sometimes
(re-find #"(?s)(^.*)\s+\[[\d-]+\]$") ; status code isn't useful and makes unit tests hard to write so strip it off
second) ; (?s) = Pattern.DOTALL - tell regex `.` to match newline characters as well
(.getMessage e))]
(throw (Exception. message)))))))
(defn process-and-run
"Process and run a query and return results."
[{:keys [type] :as query}]
(case (keyword type)
:native (native/process-and-run query)
:query (process-structured query)))
;; # IMPLEMENTATION
;; ## Query Clause Processors
(defmulti apply-form
"Given a Query clause like
{:aggregation [\"count\"]}
call the matching implementation which should either return `nil` or translate it into a korma clause like
(aggregate (count :*) :count)
An implementation of `apply-form` may optionally return a vector of several forms to insert into the generated korma `select` form."
(fn [[clause-name _]] clause-name))
(defmethod apply-form :default [form]) ;; nothing
;;; ## Formatting
(defprotocol IGenericSQLFormattable
(formatted [this] [this include-as?]))
......@@ -143,48 +80,48 @@
(formatted this false))
([{value :value, {unit :unit} :field} _]
;; prevent Clojure from converting this to #inst literal, which is a util.date
((:date (:driver *query*)) unit `(Timestamp/valueOf ~(.toString value)))))
((:date (:driver *query*)) unit value)))
RelativeDateTimeValue
(formatted
([this]
(formatted this false))
([{:keys [amount unit], {field-unit :unit} :field} _]
(let [driver (:driver *query*)]
((:date driver) field-unit (if (zero? amount)
(sqlfn :NOW)
((:date-interval driver) unit amount)))))))
(let [{:keys [date date-interval]} (:driver *query*)]
(date field-unit (if (zero? amount)
(k/sqlfn :NOW)
(date-interval unit amount)))))))
;;; ## Clause Handlers
(defmethod apply-form :aggregation [[_ {:keys [aggregation-type field]}]]
(defn- apply-aggregation [korma-query {{:keys [aggregation-type field]} :aggregation}]
(if-not field
;; aggregation clauses w/o a Field
(case aggregation-type
:rows nil ; don't need to do anything special for `rows` - `select` selects all rows by default
:count `(aggregate (~'count :*) :count))
:rows korma-query ; don't need to do anything special for `rows` - `select` selects all rows by default
:count (k/aggregate korma-query (count :*) :count))
;; aggregation clauses with a Field
(let [field (formatted field)]
(case aggregation-type
:avg `(aggregate (~'avg ~field) :avg)
:count `(aggregate (~'count ~field) :count)
:distinct `(aggregate (~'count (sqlfn :DISTINCT ~field)) :count)
:stddev `(fields [(sqlfn :stddev ~field) :stddev])
:sum `(aggregate (~'sum ~field) :sum)))))
(defmethod apply-form :breakout [[_ fields]]
`[ ;; Group by all the breakout fields
(group ~@(map formatted fields))
;; Add fields form only for fields that weren't specified in :fields clause -- we don't want to include it twice, or korma will barf
(fields ~@(->> fields
(filter (partial (complement contains?) (set (:fields (:query *query*)))))
(map (u/rpartial formatted :include-as))))])
(defmethod apply-form :fields [[_ fields]]
`(fields ~@(map (u/rpartial formatted :include-as) fields)))
:avg (k/aggregate korma-query (avg field) :avg)
:count (k/aggregate korma-query (count field) :count)
:distinct (k/aggregate korma-query (count (k/sqlfn :DISTINCT field)) :count)
:stddev (k/fields korma-query [(k/sqlfn :STDDEV field) :stddev])
:sum (k/aggregate korma-query (sum field) :sum)))))
(defn- apply-breakout [korma-query {fields :breakout}]
(-> korma-query
;; Group by all the breakout fields
((partial apply k/group) (map formatted fields))
;; Add fields form only for fields that weren't specified in :fields clause -- we don't want to include it twice, or korma will barf
((partial apply k/fields) (->> fields
(filter (partial (complement contains?) (set (:fields (:query *query*)))))
(map (u/rpartial formatted :include-as))))))
(defn- apply-fields [korma-query {fields :fields}]
(apply k/fields korma-query (for [field fields]
(formatted field :include-as))))
(defn- filter-subclause->predicate
"Given a filter SUBCLAUSE, return a Korma filter predicate form for use in korma `where`."
......@@ -192,10 +129,8 @@
(if (= filter-type :inside)
;; INSIDE filter subclause
(let [{:keys [lat lon]} filter]
(list 'and {(formatted (:field lat)) ['< (formatted (:max lat))]}
{(formatted (:field lat)) ['> (formatted (:min lat))]}
{(formatted (:field lon)) ['< (formatted (:max lon))]}
{(formatted (:field lon)) ['> (formatted (:min lon))]}))
(kfns/pred-and {(formatted (:field lat)) ['between [(formatted (:min lat)) (formatted (:max lat))]]}
{(formatted (:field lon)) ['between [(formatted (:min lon)) (formatted (:max lon))]]}))
;; all other filter subclauses
(let [field (formatted (:field filter))
......@@ -216,62 +151,93 @@
(defn- filter-clause->predicate [{:keys [compound-type subclauses], :as clause}]
(case compound-type
:and `(~'and ~@(map filter-clause->predicate subclauses))
:or `(~'or ~@(map filter-clause->predicate subclauses))
:and (apply kfns/pred-and (map filter-clause->predicate subclauses))
:or (apply kfns/pred-or (map filter-clause->predicate subclauses))
nil (filter-subclause->predicate clause)))
(defmethod apply-form :filter [[_ clause]]
`(where ~(filter-clause->predicate clause)))
(defmethod apply-form :join-tables [[_ join-tables]]
(vec (for [{:keys [table-name pk-field source-field]} join-tables]
`(join ~table-name
(~'= ~(keyword (format "%s.%s" (:name (:source-table (:query *query*))) (:field-name source-field)))
~(keyword (format "%s.%s" table-name (:field-name pk-field))))))))
(defmethod apply-form :limit [[_ value]]
`(limit ~value))
(defmethod apply-form :order-by [[_ subclauses]]
(vec (for [{:keys [field direction]} subclauses]
`(order ~(formatted field)
~(case direction
:ascending :ASC
:descending :DESC)))))
;; TODO - page can be preprocessed away -- converted to a :limit clause and an :offset clause
;; implement this at some point.
(defmethod apply-form :page [[_ {:keys [items page]}]]
{:pre [(integer? items)
(> items 0)
(integer? page)
(> page 0)]}
`[(limit ~items)
(offset ~(* items (- page 1)))])
;; ## Debugging Functions (Internal)
(defn- apply-filter [korma-query {clause :filter}]
(k/where korma-query (filter-clause->predicate clause)))
(defn- apply-join-tables [korma-query {join-tables :join-tables, {source-table-name :name} :source-table}]
(loop [korma-query korma-query, [{:keys [table-name pk-field source-field]} & more] join-tables]
(let [korma-query (k/join korma-query table-name
(= (keyword (format "%s.%s" source-table-name (:field-name source-field)))
(keyword (format "%s.%s" table-name (:field-name pk-field)))))]
(if (seq more)
(recur korma-query more)
korma-query))))
(defn- apply-limit [korma-query {value :limit}]
(k/limit korma-query value))
(defn- apply-order-by [korma-query {subclauses :order-by}]
(loop [korma-query korma-query, [{:keys [field direction]} & more] subclauses]
(let [korma-query (k/order korma-query (formatted field) (case direction
:ascending :ASC
:descending :DESC))]
(if (seq more)
(recur korma-query more)
korma-query))))
(defn- apply-page [korma-query {{:keys [items page]} :page}]
(-> korma-query
(k/limit items)
(k/offset (* items (dec page)))))
;;
(defn- log-korma-form
[korma-form]
(when-not qp/*disable-qp-logging*
(log/debug
(u/format-color 'green "\n\nKORMA FORM: 😏\n%s" (->> (nth korma-form 2) ; korma form is wrapped in a let clause. Discard it
(walk/prewalk (fn [form] ; strip korma.core/ qualifications from symbols in the form
(cond ; to remove some of the clutter
(symbol? form) (symbol (name form))
(keyword? form) (keyword (name form))
:else form)))
(u/pprint-to-str)))
(u/format-color 'blue "\nSQL: 😈\n%s\n" (-> (eval (let [[let-form binding-form & body] korma-form] ; wrap the (select ...) form in a sql-only clause
`(~let-form ~binding-form ; has to go there to work correctly
(sql-only ~@body))))
(s/replace #"\sFROM" "\nFROM") ; add newlines to the SQL to make it more readable
(s/replace #"\sLEFT JOIN" "\nLEFT JOIN")
(s/replace #"\sWHERE" "\nWHERE")
(s/replace #"\sGROUP BY" "\nGROUP BY")
(s/replace #"\sORDER BY" "\nORDER BY")
(s/replace #"\sLIMIT" "\nLIMIT"))))))
(u/format-color 'blue "\nSQL: 😈\n%s\n" (-> (k/as-sql korma-form)
(s/replace #"\sFROM" "\nFROM") ; add newlines to the SQL to make it more readable
(s/replace #"\sLEFT JOIN" "\nLEFT JOIN")
(s/replace #"\sWHERE" "\nWHERE")
(s/replace #"\sGROUP BY" "\nGROUP BY")
(s/replace #"\sORDER BY" "\nORDER BY")
(s/replace #"\sLIMIT" "\nLIMIT"))))))
(defn process-structured
"Convert QUERY into a korma `select` form, execute it, and annotate the results."
[{{:keys [source-table] :as query} :query, driver :driver, database :database, :as outer-query}]
(binding [*query* outer-query]
(try
(let [entity (korma-entity database source-table)
timezone (driver/report-timezone)
korma-query (cond-> (k/select* entity)
(:aggregation query) (apply-aggregation query)
(:breakout query) (apply-breakout query)
(:fields query) (apply-fields query)
(:filter query) (apply-filter query)
(:join-tables query) (apply-join-tables query)
(:limit query) (apply-limit query)
(:order-by query) (apply-order-by query)
(:page query) (apply-page query))]
;; log query
(when (config/config-bool :mb-db-logging)
(log-korma-form korma-query))
;; execute query
(kdb/with-db (:db entity)
(if (and (seq timezone)
(contains? (:features driver) :set-timezone))
(kdb/transaction
(k/exec-raw ((:timezone->set-timezone-sql driver) timezone))
(k/exec korma-query))
(k/exec korma-query))))
(catch java.sql.SQLException e
(let [^String message (or (->> (.getMessage e) ; error message comes back like "Error message ... [status-code]" sometimes
(re-find #"(?s)(^.*)\s+\[[\d-]+\]$") ; status code isn't useful and makes unit tests hard to write so strip it off
second) ; (?s) = Pattern.DOTALL - tell regex `.` to match newline characters as well
(.getMessage e))]
(throw (Exception. message)))))))
(defn process-and-run
"Process and run a query and return results."
[{:keys [type] :as query}]
(case (keyword type)
:native (native/process-and-run query)
:query (process-structured query)))
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment