Skip to content
Snippets Groups Projects
Unverified Commit f0d17f1f authored by Cam Saul's avatar Cam Saul
Browse files

Merge branch 'master' into dash-filter-search-widget

parents 6b9ea6a2 723849a9
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Showing
with 157 additions and 170 deletions
docs/users-guide/10-pulses.md
+ 3
− 3
View file @ f0d17f1f
......@@ -23,15 +23,15 @@ When you select a saved question, Metabase will show you a preview of how it’l
#### Attaching a .csv or .xls with results
You can also optionally include the results of a saved question in an emailed pulse as a .csv or .xls file attachment. Just click the paperclip icon on an included saved question to add the attachment. Click the paperclip again to remove the attachment.
![Attach button](images/pulses/attach-button.png)
![Attach button](images/pulses/attachments/attach-button.png)
Choose between a .csv or .xls file by clicking on the text buttons:
![Attached](images/pulses/attached.png)
![Attached](images/pulses/attachments/attached.png)
Your attachments will be included in your emailed pulse just like a regular email attachment:
![Email attachment](images/pulses/email.png)
![Email attachment](images/pulses/attachments/email.png)
#### Limitations
Currently, there are a few restrictions on what kinds of saved questions you can put into a pulse:
......
frontend/src/metabase/components/NotFound.jsx
+ 1
− 1
View file @ f0d17f1f
......@@ -17,7 +17,7 @@ export default class NotFound extends Component {
<div className="p1">{t`Ask a new question.`}</div>
</Link>
<span className="mx2">{t`or`}</span>
<a className="Button Button--withIcon" target="_blank" href="http://tv.giphy.com/kitten">
<a className="Button Button--withIcon" target="_blank" href="https://giphy.com/tv/search/kitten">
<div className="p1 flex align-center relative">
<span className="h2">😸</span>
<span className="ml1">{t`Take a kitten break.`}</span>
......
src/metabase/driver/bigquery.clj
+ 68
− 109
View file @ f0d17f1f
......@@ -23,7 +23,9 @@
[metabase.models
[database :refer [Database]]
[field :as field]]
[metabase.query-processor.util :as qputil]
[metabase.query-processor
[annotate :as annotate]
[util :as qputil]]
[metabase.util.honeysql-extensions :as hx]
[toucan.db :as db])
(:import com.google.api.client.googleapis.auth.oauth2.GoogleCredential
......@@ -32,7 +34,7 @@
TableList TableList$Tables TableReference TableRow TableSchema]
java.sql.Time
[java.util Collections Date]
[metabase.query_processor.interface DateTimeValue TimeValue Value]))
[metabase.query_processor.interface AggregationWithField AggregationWithoutField DateTimeValue Expression TimeValue Value]))
(defrecord BigQueryDriver []
clojure.lang.Named
......@@ -300,12 +302,48 @@
:rows (for [row rows]
(mapv row columns))}))
;; From the dox: Fields must contain only letters, numbers, and underscores, start with a letter or underscore, and be
;; at most 128 characters long.
(defn- format-custom-field-name ^String [^String custom-field-name]
(let [replaced-str (-> (str/trim custom-field-name)
(str/replace #"[^\w\d_]" "_")
(str/replace #"(^\d)" "_$1"))]
(subs replaced-str 0 (min 128 (count replaced-str)))))
(defn- agg-or-exp? [x]
(or (instance? Expression x)
(instance? AggregationWithField x)
(instance? AggregationWithoutField x)))
(defn- bg-aggregate-name [aggregate]
(-> aggregate annotate/aggregation-name format-custom-field-name))
(defn- pre-alias-aggregations
"Expressions are not allowed in the order by clauses of a BQ query. To sort by a custom expression, that custom
expression must be aliased from the order by. This code will find the aggregations and give them a name if they
don't already have one. This name can then be used in the order by if one is present."
[query]
(let [aliases (atom {})]
(walk/postwalk (fn [maybe-agg]
(if-let [exp-name (and (agg-or-exp? maybe-agg)
(bg-aggregate-name maybe-agg))]
(if-let [usage-count (get @aliases exp-name)]
(let [new-custom-name (str exp-name "_" (inc usage-count))]
(swap! aliases assoc
exp-name (inc usage-count)
new-custom-name 1)
(assoc maybe-agg :custom-name new-custom-name))
(do
(swap! aliases assoc exp-name 1)
(assoc maybe-agg :custom-name exp-name)))
maybe-agg))
query)))
(defn- mbql->native [{{{:keys [dataset-id]} :details, :as database} :database, {{table-name :name} :source-table} :query, :as outer-query}]
{:pre [(map? database) (seq dataset-id) (seq table-name)]}
(binding [sqlqp/*query* outer-query]
(let [honeysql-form (honeysql-form outer-query)
sql (honeysql-form->sql honeysql-form)]
{:query sql
(let [aliased-query (pre-alias-aggregations outer-query)]
(binding [sqlqp/*query* aliased-query]
{:query (-> aliased-query honeysql-form honeysql-form->sql)
:table-name table-name
:mbql? true})))
......@@ -341,18 +379,25 @@
(sqlqp/->honeysql driver)
hx/->time))
(defn- field->alias [{:keys [^String schema-name, ^String field-name, ^String table-name, ^Integer index, field], :as this}]
(defn- field->alias [driver {:keys [^String schema-name, ^String field-name, ^String table-name, ^Integer index, field], :as this}]
{:pre [(map? this) (or field
index
(and (seq schema-name) (seq field-name) (seq table-name))
(log/error "Don't know how to alias: " this))]}
(cond
field (recur field) ; type/DateTime
index (name (let [{{aggregations :aggregation} :query} sqlqp/*query*
{ag-type :aggregation-type} (nth aggregations index)]
(if (= ag-type :distinct)
:count
ag-type)))
field (recur driver field) ; type/DateTime
index (let [{{aggregations :aggregation} :query} sqlqp/*query*
{ag-type :aggregation-type :as agg} (nth aggregations index)]
(cond
(= ag-type :distinct)
"count"
(instance? Expression agg)
(:custom-name agg)
:else
(name ag-type)))
:else (str schema-name \. table-name \. field-name)))
;; TODO - Making 2 DB calls for each field to fetch its dataset is inefficient and makes me cry, but this method is
......@@ -362,91 +407,13 @@
dataset (:dataset-id (db/select-one-field :details Database, :id db-id))]
(hsql/raw (apply format "[%s.%s.%s]" dataset (field/qualified-name-components field)))))
;; We have to override the default SQL implementations of breakout and order-by because BigQuery propogates casting
;; functions in SELECT
;; BAD:
;; SELECT msec_to_timestamp([sad_toucan_incidents.incidents.timestamp]) AS [sad_toucan_incidents.incidents.timestamp],
;; count(*) AS [count]
;; FROM [sad_toucan_incidents.incidents]
;; GROUP BY msec_to_timestamp([sad_toucan_incidents.incidents.timestamp])
;; ORDER BY msec_to_timestamp([sad_toucan_incidents.incidents.timestamp]) ASC
;; LIMIT 10
;;
;; GOOD:
;; SELECT msec_to_timestamp([sad_toucan_incidents.incidents.timestamp]) AS [sad_toucan_incidents.incidents.timestamp],
;; count(*) AS [count]
;; FROM [sad_toucan_incidents.incidents]
;; GROUP BY [sad_toucan_incidents.incidents.timestamp]
;; ORDER BY [sad_toucan_incidents.incidents.timestamp] ASC
;; LIMIT 10
(defn- deduplicate-aliases
"Given a sequence of aliases, return a sequence where duplicate aliases have been appropriately suffixed.
(deduplicate-aliases [\"sum\" \"count\" \"sum\" \"avg\" \"sum\" \"min\"])
;; -> [\"sum\" \"count\" \"sum_2\" \"avg\" \"sum_3\" \"min\"]"
[aliases]
(loop [acc [], alias->use-count {}, [alias & more, :as aliases] aliases]
(let [use-count (get alias->use-count alias)]
(cond
(empty? aliases) acc
(not alias) (recur (conj acc alias) alias->use-count more)
(not use-count) (recur (conj acc alias) (assoc alias->use-count alias 1) more)
:else (let [new-count (inc use-count)
new-alias (str alias "_" new-count)]
(recur (conj acc new-alias) (assoc alias->use-count alias new-count, new-alias 1) more))))))
(defn- select-subclauses->aliases
"Return a vector of aliases used in HoneySQL SELECT-SUBCLAUSES.
(For clauses that aren't aliased, `nil` is returned as a placeholder)."
[select-subclauses]
(for [subclause select-subclauses]
(when (and (vector? subclause)
(= 2 (count subclause)))
(second subclause))))
(defn update-select-subclause-aliases
"Given a vector of HoneySQL SELECT-SUBCLAUSES and a vector of equal length of NEW-ALIASES,
return a new vector with combining the original `SELECT` subclauses with the new aliases.
Subclauses that are not aliased are not modified; they are given a placeholder of `nil` in the NEW-ALIASES vector.
(update-select-subclause-aliases [[:user_id \"user_id\"] :venue_id]
[\"user_id_2\" nil])
;; -> [[:user_id \"user_id_2\"] :venue_id]"
[select-subclauses new-aliases]
(for [[subclause new-alias] (partition 2 (interleave select-subclauses new-aliases))]
(if-not new-alias
subclause
[(first subclause) new-alias])))
(defn- deduplicate-select-aliases
"Replace duplicate aliases in SELECT-SUBCLAUSES with appropriately suffixed aliases.
BigQuery doesn't allow duplicate aliases in `SELECT` statements; a statement like `SELECT sum(x) AS sum, sum(y) AS
sum` is invalid. (See #4089) To work around this, we'll modify the HoneySQL aliases to make sure the same one isn't
used twice by suffixing duplicates appropriately.
(We'll generate SQL like `SELECT sum(x) AS sum, sum(y) AS sum_2` instead.)"
[select-subclauses]
(let [aliases (select-subclauses->aliases select-subclauses)
deduped (deduplicate-aliases aliases)]
(update-select-subclause-aliases select-subclauses deduped)))
(defn- apply-aggregation
"BigQuery's implementation of `apply-aggregation` just hands off to the normal Generic SQL implementation, but calls
`deduplicate-select-aliases` on the results."
[driver honeysql-form query]
(-> (sqlqp/apply-aggregation driver honeysql-form query)
(update :select deduplicate-select-aliases)))
(defn- field->breakout-identifier [field]
(hsql/raw (str \[ (field->alias field) \])))
(defn- field->breakout-identifier [driver field]
(hsql/raw (str \[ (field->alias driver field) \])))
(defn- apply-breakout [driver honeysql-form {breakout-fields :breakout, fields-fields :fields}]
(-> honeysql-form
;; Group by all the breakout fields
((partial apply h/group) (map field->breakout-identifier breakout-fields))
((partial apply h/group) (map #(field->breakout-identifier driver %) breakout-fields))
;; Add fields form only for fields that weren't specified in :fields clause -- we don't want to include it
;; twice, or HoneySQL will barf
((partial apply h/merge-select) (for [field breakout-fields
......@@ -465,11 +432,11 @@
(recur honeysql-form more)
honeysql-form))))
(defn- apply-order-by [honeysql-form {subclauses :order-by}]
(defn- apply-order-by [driver honeysql-form {subclauses :order-by}]
(loop [honeysql-form honeysql-form, [{:keys [field direction]} & more] subclauses]
(let [honeysql-form (h/merge-order-by honeysql-form [(field->breakout-identifier field) (case direction
:ascending :asc
:descending :desc)])]
(let [honeysql-form (h/merge-order-by honeysql-form [(field->breakout-identifier driver field) (case direction
:ascending :asc
:descending :desc)])]
(if (seq more)
(recur honeysql-form more)
honeysql-form))))
......@@ -477,13 +444,6 @@
(defn- string-length-fn [field-key]
(hsql/call :length field-key))
;; From the dox: Fields must contain only letters, numbers, and underscores, start with a letter or underscore, and be
;; at most 128 characters long.
(defn- format-custom-field-name ^String [^String custom-field-name]
(str/join (take 128 (-> (str/trim custom-field-name)
(str/replace #"[^\w\d_]" "_")
(str/replace #"(^\d)" "_$1")))))
(defn- date-interval [driver unit amount]
(sqlqp/->honeysql driver (u/relative-date unit amount)))
......@@ -497,16 +457,15 @@
(u/strict-extend BigQueryDriver
sql/ISQLDriver
(merge (sql/ISQLDriverDefaultsMixin)
{:apply-aggregation apply-aggregation
:apply-breakout apply-breakout
{:apply-breakout apply-breakout
:apply-join-tables (u/drop-first-arg apply-join-tables)
:apply-order-by (u/drop-first-arg apply-order-by)
:apply-order-by apply-order-by
;; these two are actually not applicable since we don't use JDBC
:column->base-type (constantly nil)
:connection-details->spec (constantly nil)
:current-datetime-fn (constantly :%current_timestamp)
:date (u/drop-first-arg date)
:field->alias (u/drop-first-arg field->alias)
:field->alias field->alias
:field->identifier (u/drop-first-arg field->identifier)
;; we want identifiers quoted [like].[this] initially (we have to convert them to [like.this] before
;; executing)
......
src/metabase/driver/druid/query_processor.clj
+ 12
− 13
View file @ f0d17f1f
......@@ -519,14 +519,13 @@
both. Defaults to being `inclusive` (e.g. `<=` instead of `<`) but specify option `inclusive?` to change this."
[field & {:keys [lower upper inclusive?]
:or {inclusive? true}}]
(u/prog1 {:type :bound
:ordering :numeric
:dimension (->rvalue field)
:lower (num (->rvalue lower))
:upper (num (->rvalue upper))
:lowerStrict (not inclusive?)
:upperStrict (not inclusive?)}
(println "inclusive?" inclusive? (u/pprint-to-str 'blue <>))))
{:type :bound
:ordering :numeric
:dimension (->rvalue field)
:lower (num (->rvalue lower))
:upper (num (->rvalue upper))
:lowerStrict (not inclusive?)
:upperStrict (not inclusive?)})
(defn- check-filter-fields [filter-type & fields]
(doseq [field fields]
......@@ -576,10 +575,10 @@
:= (filter:= field value)
:!= (filter:not (filter:= field value))
:< (filter:bound field, :lower value, :inclusive? false)
:> (filter:bound field, :upper value, :inclusive? false)
:<= (filter:bound field, :lower value)
:>= (filter:bound field, :upper value)))
:< (filter:bound field, :upper value, :inclusive? false)
:> (filter:bound field, :lower value, :inclusive? false)
:<= (filter:bound field, :upper value)
:>= (filter:bound field, :lower value)))
(catch Throwable e
(log/warn (.getMessage e))))))
......@@ -588,7 +587,7 @@
(case compound-type
:and {:type :and, :fields (filterv identity (map parse-filter-clause:filter subclauses))}
:or {:type :or, :fields (filterv identity (map parse-filter-clause:filter subclauses))}
:not (when-let [subclause (parse-filter-subclause:filter subclause)]
:not (when-let [subclause (parse-filter-clause:filter subclause)]
(filter:not subclause))
nil (parse-filter-subclause:filter clause)))
......
src/metabase/driver/generic_sql/query_processor.clj
+ 37
− 31
View file @ f0d17f1f
......@@ -129,36 +129,6 @@
(hx/* bin-width)
(hx/+ min-value))))
;; e.g. the ["aggregation" 0] fields we allow in order-by
(defmethod ->honeysql [Object AgFieldRef]
[_ {index :index}]
(let [{:keys [aggregation-type]} (aggregation-at-index index)]
;; For some arcane reason we name the results of a distinct aggregation "count",
;; everything else is named the same as the aggregation
(if (= aggregation-type :distinct)
:count
aggregation-type)))
(defmethod ->honeysql [Object Value]
[driver {:keys [value]}]
(->honeysql driver value))
(defmethod ->honeysql [Object DateTimeValue]
[driver {{unit :unit} :field, value :value}]
(sql/date driver unit (->honeysql driver value)))
(defmethod ->honeysql [Object RelativeDateTimeValue]
[driver {:keys [amount unit], {field-unit :unit} :field}]
(sql/date driver field-unit (if (zero? amount)
(sql/current-datetime-fn driver)
(driver/date-interval driver unit amount))))
(defmethod ->honeysql [Object TimeValue]
[driver {:keys [value]}]
(->honeysql driver value))
;;; ## Clause Handlers
(defn- aggregation->honeysql
"Generate the HoneySQL form for an aggregation."
[driver aggregation-type field]
......@@ -181,7 +151,7 @@
(->honeysql driver field))))
;; TODO - can't we just roll this into the ->honeysql method for `expression`?
(defn- expression-aggregation->honeysql
(defn expression-aggregation->honeysql
"Generate the HoneySQL form for an expression aggregation."
[driver expression]
(->honeysql driver
......@@ -193,6 +163,42 @@
(:aggregation-type arg) (aggregation->honeysql driver (:aggregation-type arg) (:field arg))
(:operator arg) (expression-aggregation->honeysql driver arg)))))))
;; e.g. the ["aggregation" 0] fields we allow in order-by
(defmethod ->honeysql [Object AgFieldRef]
[driver {index :index}]
(let [{:keys [aggregation-type] :as aggregation} (aggregation-at-index index)]
(cond
;; For some arcane reason we name the results of a distinct aggregation "count",
;; everything else is named the same as the aggregation
(= aggregation-type :distinct)
:count
(instance? Expression aggregation)
(expression-aggregation->honeysql driver aggregation)
:else
aggregation-type)))
(defmethod ->honeysql [Object Value]
[driver {:keys [value]}]
(->honeysql driver value))
(defmethod ->honeysql [Object DateTimeValue]
[driver {{unit :unit} :field, value :value}]
(sql/date driver unit (->honeysql driver value)))
(defmethod ->honeysql [Object RelativeDateTimeValue]
[driver {:keys [amount unit], {field-unit :unit} :field}]
(sql/date driver field-unit (if (zero? amount)
(sql/current-datetime-fn driver)
(driver/date-interval driver unit amount))))
(defmethod ->honeysql [Object TimeValue]
[driver {:keys [value]}]
(->honeysql driver value))
;;; ## Clause Handlers
(defn- apply-expression-aggregation [driver honeysql-form expression]
(h/merge-select honeysql-form [(expression-aggregation->honeysql driver expression)
(hx/escape-dots (driver/format-custom-field-name driver (annotate/aggregation-name expression)))]))
......
test/metabase/driver/bigquery_test.clj
+ 27
− 13
View file @ f0d17f1f
......@@ -13,7 +13,7 @@
[metabase.test
[data :as data]
[util :as tu]]
[metabase.test.data.datasets :refer [expect-with-engine]]))
[metabase.test.data.datasets :refer [expect-with-engine do-with-engine]]))
(def ^:private col-defaults
{:remapped_to nil, :remapped_from nil})
......@@ -76,24 +76,38 @@
["field-id" (data/id :checkins :venue_id)]]]
"User ID Plus Venue ID"]]}})))
(defn- aggregation-names [query-map]
(->> query-map
:aggregation
(map :custom-name)))
(defn- pre-alias-aggregations' [query-map]
(binding [qpi/*driver* (driver/engine->driver :bigquery)]
(aggregation-names (#'bigquery/pre-alias-aggregations query-map))))
(defn- agg-query-map [aggregations]
(-> {}
(ql/source-table 1)
(ql/aggregation aggregations)))
;; make sure BigQuery can handle two aggregations with the same name (#4089)
(expect
["sum" "count" "sum_2" "avg" "sum_3" "min"]
(#'bigquery/deduplicate-aliases ["sum" "count" "sum" "avg" "sum" "min"]))
(pre-alias-aggregations' (agg-query-map [(ql/sum (ql/field-id 2))
(ql/count (ql/field-id 2))
(ql/sum (ql/field-id 2))
(ql/avg (ql/field-id 2))
(ql/sum (ql/field-id 2))
(ql/min (ql/field-id 2))])))
(expect
["sum" "count" "sum_2" "avg" "sum_2_2" "min"]
(#'bigquery/deduplicate-aliases ["sum" "count" "sum" "avg" "sum_2" "min"]))
(expect
["sum" "count" nil "sum_2"]
(#'bigquery/deduplicate-aliases ["sum" "count" nil "sum"]))
(expect
[[:user_id "user_id_2"] :venue_id]
(#'bigquery/update-select-subclause-aliases [[:user_id "user_id"] :venue_id]
["user_id_2" nil]))
(pre-alias-aggregations' (agg-query-map [(ql/sum (ql/field-id 2))
(ql/count (ql/field-id 2))
(ql/sum (ql/field-id 2))
(ql/avg (ql/field-id 2))
(assoc (ql/sum (ql/field-id 2)) :custom-name "sum_2")
(ql/min (ql/field-id 2))])))
(expect-with-engine :bigquery
{:rows [[7929 7929]], :columns ["sum" "sum_2"]}
......
test/metabase/query_processor_test/expression_aggregations_test.clj
+ 9
− 0
View file @ f0d17f1f
......@@ -150,6 +150,15 @@
(ql/aggregation (ql/+ 1 (ql/count)))
(ql/breakout $price)))))
;; Sorting by an un-named aggregate expression
(datasets/expect-with-engines (non-timeseries-engines-with-feature :expression-aggregations)
[[1 2] [2 2] [12 2] [4 4] [7 4] [10 4] [11 4] [8 8]]
(format-rows-by [int int]
(rows (data/run-query users
(ql/aggregation (ql/* (ql/count) 2))
(ql/breakout (ql/datetime-field $last_login :month-of-year))
(ql/order-by (ql/asc (ql/aggregation 0)))))))
;; aggregation with math inside the aggregation :scream_cat:
(datasets/expect-with-engines (non-timeseries-engines-with-feature :expression-aggregations)
[[1 44]
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment