Fix coloring, filters. Support native queries

src/metabase/automagic_dashboards/core.clj

@@ -488,8 +488,13 @@
                               :id)
                         (->> source
                              :result_metadata
-                             (map (comp #(classify/run-classifiers % {}) field/map->FieldInstance))
-                             constantly ))]
+                             (map (fn [field]
+                                    (-> field
+                                        (update :base_type keyword)
+                                        (update :special_type keyword)
+                                        field/map->FieldInstance
+                                        (classify/run-classifiers {}))))
+                             constantly))]
     (as-> {:source       (assoc source :fields (table->fields source))
            :tables       (map #(assoc % :fields (table->fields %)) tables)
            :database     (:database root)
@@ -532,9 +537,15 @@
         cards     (make-cards context rule)]
     (when cards
       [(assoc dashboard
-         :filters filters
-         :cards   cards
-         :context context)
+         :filters  filters
+         :cards    cards
+         :context  context
+         :fieldset (->> context
+                        :tables
+                        (mapcat :fields)
+                        (map (fn [field]
+                               [((some-fn :id :name) field) field]))
+                        (into {})))
        rule])))
 
 (def ^:private ^:const ^Long max-related 6)
@@ -644,10 +655,10 @@
   (comp codec/base64-encode codecs/str->bytes json/encode))
 
 (def ^:private ^{:arglists '([card-or-question])} nested-query?
-  (comp string? :source_table :query :dataset_query))
+  (comp (every-pred string? #(str/starts-with? % "card__")) :source_table :query :dataset_query))
 
 (def ^:private ^{:arglists '([card-or-question])} native-query?
-  (comp #{:native} qp.util/normalize-token :type :query :dataset_query))
+  (comp #{:native} qp.util/normalize-token :type :dataset_query))
 
 (def ^:private ^{:arglists '([card-or-question])} source-question
   (comp Card #(Integer/parseInt %) second #(str/split % #"__") :source_table :query :dataset_query))
@@ -656,18 +667,16 @@
   [card {:keys [cell-query] :as opts}]
   (if (or (table-like? card)
           cell-query)
-    (let [source (if (nested-query? card)
-                   (-> card source-question (assoc :entity_type :entity/GenericTable))
-                   (-> card :table_id Table))]
+    (let [source (cond
+                   (nested-query? card) (-> card
+                                            source-question
+                                            (assoc :entity_type :entity/GenericTable))
+                   (native-query? card) (-> card (assoc :entity_type :entity/GenericTable))
+                   :else                (-> card :table_id Table))]
       (automagic-dashboard
        (merge {:entity       source
                :source       source
-               :query-filter (cond-> (-> card :dataset_query :query :filter)
-                               (nested-query? card) (merge-filter-clauses (-> card
-                                                                              source-question
-                                                                              :dataset_query
-                                                                              :query
-                                                                              :filter)))
+               :query-filter (-> card :dataset_query :query :filter)
                :database     (:database_id card)
                :full-name    (str (:name card) " question")
                :url          (if cell-query
@@ -683,14 +692,15 @@
   [query {:keys [cell-query] :as opts}]
   (if (or (table-like? query)
           (:cell-query opts))
-    (let [source (if (nested-query? query)
-                   (-> query source-question (assoc :entity_type :entity/GenericTable))
-                   (-> query :table_id Table))]
+    (let [source (cond
+                   (nested-query? query) (-> query
+                                             source-question
+                                             (assoc :entity_type :entity/GenericTable))
+                   (native-query? query) (-> query (assoc :entity_type :entity/GenericTable))
+                   :else                 (-> query :table-id Table))]
       (automagic-dashboard
        (merge {:entity       source
                :source       source
-               :query-filter (when (nested-query? query)
-                               (-> source :dataset_query :query :filter))
                :database     (:database-id query)
                :full-name    (if (nested-query? query)
                                (:name source)
@@ -700,7 +710,7 @@
                                        (endocde-base64-json (:dataset_query query))
                                        (endocde-base64-json cell-query))
                                (format "%sadhoc/%s" public-endpoint
-                                                   (endocde-base64-json query)))
+                                       (endocde-base64-json query)))
                :rules-prefix "table"}
               (update opts :cell-query merge-filter-clauses (-> query
                                                                 :dataset_query

src/metabase/automagic_dashboards/filters.clj

@@ -9,45 +9,74 @@
             [schema.core :as s]
             [toucan.db :as db]))
 
-(def ^:private FieldIdForm
+(def ^:private FieldReference
   [(s/one (s/constrained su/KeywordOrString
-                         (comp #{:field-id :fk->} qp.util/normalize-token))
+                         (comp #{:field-id :fk-> :field-literal} qp.util/normalize-token))
           "head")
    s/Any])
 
-(def ^{:arglists '([form])} field-form?
-  "Is given form an MBQL field form?"
-  (complement (s/checker FieldIdForm)))
+(def ^:private ^{:arglists '([form])} field-reference?
+  "Is given form an MBQL field reference?"
+  (complement (s/checker FieldReference)))
+
+(defmulti
+  ^{:doc "Extract field ID from a given field reference form."
+    :arglists '([op & args])}
+  field-reference->id (comp qp.util/normalize-token first))
+
+(defmethod field-reference->id :field-id
+  [[_ id]]
+  id)
+
+(defmethod field-reference->id :fk->
+  [[_ _ id]]
+  id)
+
+(defmethod field-reference->id :field-literal
+  [[_ name _]]
+  name)
 
 (defn collect-field-references
   "Collect all field references (`[:field-id]` or `[:fk->]` forms) from a given form."
   [form]
   (->> form
        (tree-seq (some-fn sequential? map?) identity)
-       (filter field-form?)))
+       (filter field-reference?)))
+
+(def ^:private ^{:arglists '([field])} periodic-datetime?
+  (comp #{:minute-of-hour :hour-of-day :day-of-week :day-of-month :day-of-year :week-of-year
+          :month-of-year :quarter-of-year}
+        :unit))
+
+(defn- datetime?
+  [field]
+  (and (not (periodic-datetime? field))
+       (or (isa? (:base_type field) :type/DateTime)
+           (field/unix-timestamp? field))))
 
 (defn- candidates-for-filtering
-  [cards]
+  [fieldset cards]
   (->> cards
        (mapcat collect-field-references)
-       (map last)
+       (map field-reference->id)
        distinct
-       (map Field)
-       (filter (fn [{:keys [base_type special_type] :as field}]
-                 (or (isa? base_type :type/DateTime)
-                     (isa? special_type :type/Category)
-                     (field/unix-timestamp? field))))))
+       (map fieldset)
+       (filter (fn [{:keys [special_type] :as field}]
+                 (or (datetime? field)
+                     (isa? special_type :type/Category))))))
 
 (defn- build-fk-map
   [fks field]
-  (->> fks
-       (filter (comp #{(:table_id field)} :table_id :target))
-       (group-by :table_id)
-       (keep (fn [[_ [fk & fks]]]
-               ;; Bail out if there is more than one FK from the same table
-               (when (empty? fks)
-                 [(:table_id fk) [:fk-> (:id fk) (:id field)]])))
-       (into {(:table_id field) [:field-id (:id field)]})))
+  (if (:id field)
+    (->> fks
+         (filter (comp #{(:table_id field)} :table_id :target))
+         (group-by :table_id)
+         (keep (fn [[_ [fk & fks]]]
+                 ;; Bail out if there is more than one FK from the same table
+                 (when (empty? fks)
+                   [(:table_id fk) [:fk-> (:id fk) (:id field)]])))
+         (into {(:table_id field) [:field-id (:id field)]}))
+    (constantly [:field-literal (:name field) (:base_type field)])))
 
 (defn- filter-for-card
   [card field]
@@ -67,18 +96,6 @@
       (nil? (:card dashcard)) dashcard
       mappings                (update dashcard :parameter_mappings concat mappings))))
 
-
-(def ^:private ^{:arglists '([field])} periodic-datetime?
-  (comp #{:minute-of-hour :hour-of-day :day-of-week :day-of-month :day-of-year :week-of-year
-          :month-of-year :quarter-of-year}
-        :unit))
-
-(defn- datetime?
-  [field]
-  (and (not (periodic-datetime? field))
-       (or (isa? (:base_type field) :type/DateTime)
-           (field/unix-timestamp? field))))
-
 (defn- filter-type
   "Return filter type for a given field."
   [{:keys [base_type special_type] :as field}]
@@ -111,13 +128,12 @@
   ([dashboard max-filters]
    (->> dashboard
         :orderd_cards
-        candidates-for-filtering
+        (candidates-for-filtering (:fieldset dashboard))
         (add-filters dashboard max-filters)))
   ([dashboard dimensions max-filters]
    (let [fks (->> (db/select Field
                     :fk_target_field_id [:not= nil]
-                    :table_id [:in (keep (comp :table_id :card)
-                                         (:ordered_cards dashboard))])
+                    :table_id [:in (keep (comp :table_id :card) (:ordered_cards dashboard))])
                   field/with-targets)]
      (->> dimensions
           remove-unqualified
@@ -152,15 +168,16 @@
                     (t.format/parse dt)))
 
 (defn- field-reference->field
-  [field-reference]
-  (cond-> (-> field-reference collect-field-references first last Field)
+  [fieldset field-reference]
+  (cond-> (-> field-reference collect-field-references first field-reference->id fieldset)
     (-> field-reference first qp.util/normalize-token (= :datetime-field))
     (assoc :unit (-> field-reference last qp.util/normalize-token))))
 
 (defmulti
   ^{:private true
-    :arglists '([op & args])}
-  humanize-filter-value (comp qp.util/normalize-token first))
+    :arglists '([fieldset [op & args]])}
+  humanize-filter-value (fn [_ [op & args]]
+                          (qp.util/normalize-token op)))
 
 (defn- either
   [v & vs]
@@ -174,83 +191,83 @@
         :else       (recur (format "%s, %s" acc v) vs)))))
 
 (defmethod humanize-filter-value :=
-  [[_ field-reference value & values]]
-  (let [field (field-reference->field field-reference)]
+  [fieldset [_ field-reference value & values]]
+  (let [field (field-reference->field fieldset field-reference)]
     [{:field field-reference
       :value (if (datetime? field)
                (format "is on %s" (humanize-datetime value))
                (format "is %s" (apply either value values)))}]))
 
 (defmethod humanize-filter-value :not=
-  [[_ field-reference value]]
+  [_ [_ field-reference value]]
   [{:field field-reference
     :value (format "is not %s" value)}])
 
 (defmethod humanize-filter-value :>
-  [[_ field-reference value]]
-  (let [field (field-reference->field field-reference)]
+  [fieldset [_ field-reference value]]
+  (let [field (field-reference->field fieldset field-reference)]
     [{:field field-reference
       :value (if (datetime? field)
                (format "is after %s" (humanize-datetime value))
                (format "is greater than %s" value))}]))
 
 (defmethod humanize-filter-value :<
-  [[_ field-reference value]]
-  (let [field (field-reference->field field-reference)]
+  [fieldset [_ field-reference value]]
+  (let [field (field-reference->field fieldset field-reference)]
   [{:field field-reference
     :value (if (datetime? field)
              (format "is before %s" (humanize-datetime value))
              (format "is less than %s" value))}]))
 
 (defmethod humanize-filter-value :>=
-  [[_ field-reference value]]
+  [_ [_ field-reference value]]
   [{:field field-reference
     :value (format "is greater than or equal to %s" value)}])
 
 (defmethod humanize-filter-value :<=
-  [[_ field-reference value]]
-  [{:field field-reference
+  [_ [_ field-reference value]]
+  [_ {:field field-reference
     :value (format "is less than or equal to %s" value)}])
 
 (defmethod humanize-filter-value :is-null
-  [[_ field-reference]]
+  [_ [_ field-reference]]
   [{:field field-reference
     :value "is null"}])
 
 (defmethod humanize-filter-value :not-null
-  [[_ field-reference]]
+  [_ [_ field-reference]]
   [{:field field-reference
     :value "is not null"}])
 
 (defmethod humanize-filter-value :between
-  [[_ field-reference min-value max-value]]
+  [_ [_ field-reference min-value max-value]]
   [{:field field-reference
     :value (format "is between %s and %s" min-value max-value)}])
 
 (defmethod humanize-filter-value :inside
-  [[_ lat-reference lon-reference lat-max lon-min lat-min lon-max]]
+  [_ [_ lat-reference lon-reference lat-max lon-min lat-min lon-max]]
   [{:field lat-reference
     :value (format "is between %s and %s" lat-min lat-max)}
    {:field lon-reference
     :value (format "is between %s and %s" lon-min lon-max)}])
 
 (defmethod humanize-filter-value :starts-with
-  [[_ field-reference value]]
+  [_ [_ field-reference value]]
   [{:field field-reference
     :value (format "starts with %s" value)}])
 
 (defmethod humanize-filter-value :contains
-  [[_ field-reference value]]
+  [_ [_ field-reference value]]
   [{:field field-reference
     :value (format "contains %s" value)}])
 
 (defmethod humanize-filter-value :does-not-contain
-  [[_ field-reference value]]
+  [_ [_ field-reference value]]
   [{:field field-reference
     :value (format "does not contain %s" value)}])
 
 (defmethod humanize-filter-value :ends-with
-  [[_ field-reference value]]
+  [_ [_ field-reference value]]
   [{:field field-reference
     :value (format "ends with %s" value)}])
 
@@ -265,13 +282,13 @@
       (neg? n)  (format "previous %s %ss" n unit))))
 
 (defmethod humanize-filter-value :time-interval
-  [[_ field-reference n unit]]
+  [_ [_ field-reference n unit]]
   [{:field field-reference
     :value (format "is during the %s" (time-interval n unit))}])
 
 (defmethod humanize-filter-value :and
-  [[_ & clauses]]
-  (mapcat humanize-filter-value clauses))
+  [fieldset [_ & clauses]]
+  (mapcat (partial humanize-filter-value fieldset) clauses))
 
 (def ^:private unit-name (comp {:minute-of-hour  "minute of hour"
                                 :hour-of-day     "hour of day"
@@ -293,11 +310,11 @@
 
 (defn applied-filters
   "Extract fields and their values from MBQL filter clauses."
-  [filter-clause]
+  [fieldset filter-clause]
   (for [{field-reference :field value :value} (some->> filter-clause
                                                        not-empty
-                                                       humanize-filter-value)]
-    (let [field (field-reference->field field-reference)]
+                                                       (humanize-filter-value fieldset))]
+    (let [field (field-reference->field fieldset field-reference)]
       {:field    (field-name field field-reference)
        :field_id (:id field)
        :type     (filter-type field)

src/metabase/automagic_dashboards/populate.clj

@@ -68,9 +68,8 @@
                                          qp.util/normalize-token
                                          (= :count)))
                          (->> breakout
-                              (tree-seq sequential? identity)
-                              (filter magic.filters/field-form?)
-                              (map last))
+                              magic.filters/collect-field-references
+                              (map magic.filters/field-reference->id))
                          aggregation)]
         {:graph.colors (->> color-keys
                             (map (comp colors #(mod % (count colors)) hash))
@@ -234,14 +233,14 @@
 (defn create-dashboard
   "Create dashboard and populate it with cards."
   ([dashboard] (create-dashboard dashboard :all))
-  ([{:keys [title transient_title description groups filters cards refinements]} n]
+  ([{:keys [title transient_title description groups filters cards refinements fieldset]} n]
    (let [n             (cond
                          (= n :all)   (count cards)
                          (keyword? n) (Integer/parseInt (name n))
                          :else        n)
          dashboard     {:name              title
                         :transient_name    (or transient_title title)
-                        :transient_filters (magic.filters/applied-filters refinements)
+                        :transient_filters (magic.filters/applied-filters fieldset refinements)
                         :description       description
                         :creator_id        api/*current-user-id*
                         :parameters        []}