Merge branch 'xray-admin-toggle' of github.com:metabase/metabase into xray-admin-toggle

src/metabase/api/x_ray.clj

@@ -33,14 +33,16 @@
 
 (defn- max-cost
   [query computation]
-  {:query       (keyword query)
-   :computation (keyword computation)})
+  (costs/apply-global-cost-cap
+   {:query       (keyword query)
+    :computation (keyword computation)}))
 
 (api/defendpoint GET "/field/:id"
   "Get x-ray for a `Field` with ID."
   [id max_query_cost max_computation_cost]
   {max_query_cost       MaxQueryCost
    max_computation_cost MaxComputationCost}
+  (api/check-403 (costs/enable-xrays))
   (->> id
        (api/read-check Field)
        (fe/extract-features {:max-cost (max-cost max_query_cost
@@ -52,6 +54,7 @@
   [id max_query_cost max_computation_cost]
   {max_query_cost       MaxQueryCost
    max_computation_cost MaxComputationCost}
+  (api/check-403 (costs/enable-xrays))
   (->> id
        (api/read-check Table)
        (fe/extract-features {:max-cost (max-cost max_query_cost
@@ -63,6 +66,7 @@
   [id max_query_cost max_computation_cost]
   {max_query_cost       MaxQueryCost
    max_computation_cost MaxComputationCost}
+  (api/check-403 (costs/enable-xrays))
   (->> id
        (api/read-check Segment)
        (fe/extract-features {:max-cost (max-cost max_query_cost
@@ -74,6 +78,7 @@
   [id max_query_cost max_computation_cost]
   {max_query_cost       MaxQueryCost
    max_computation_cost MaxComputationCost}
+  (api/check-403 (costs/enable-xrays))
   (->> id
        (api/read-check Card)
        (fe/extract-features {:max-cost (max-cost max_query_cost
@@ -85,6 +90,7 @@
   [id1 id2 max_query_cost max_computation_cost]
   {max_query_cost       MaxQueryCost
    max_computation_cost MaxComputationCost}
+  (api/check-403 (costs/enable-xrays))
   (->> [id1 id2]
        (map #(api/read-check Field (Integer/parseInt %)))
        (apply fe/compare-features
@@ -96,6 +102,7 @@
   [id1 id2 max_query_cost max_computation_cost]
   {max_query_cost       MaxQueryCost
    max_computation_cost MaxComputationCost}
+  (api/check-403 (costs/enable-xrays))
   (->> [id1 id2]
        (map #(api/read-check Table (Integer/parseInt %)))
        (apply fe/compare-features
@@ -108,6 +115,7 @@
   [id1 id2 field max_query_cost max_computation_cost]
   {max_query_cost       MaxQueryCost
    max_computation_cost MaxComputationCost}
+  (api/check-403 (costs/enable-xrays))
   (let [{:keys [comparison constituents]}
         (->> [id1 id2]
              (map #(api/read-check Table (Integer/parseInt %)))
@@ -134,6 +142,7 @@
   [id1 id2 max_query_cost max_computation_cost]
   {max_query_cost       MaxQueryCost
    max_computation_cost MaxComputationCost}
+  (api/check-403 (costs/enable-xrays))
   (->> [id1 id2]
        (map #(api/read-check Segment (Integer/parseInt %)))
        (apply fe/compare-features
@@ -146,6 +155,7 @@
   [id1 id2 field max_query_cost max_computation_cost]
   {max_query_cost       MaxQueryCost
    max_computation_cost MaxComputationCost}
+  (api/check-403 (costs/enable-xrays))
   (let [{:keys [comparison constituents]}
         (->> [id1 id2]
              (map #(api/read-check Segment (Integer/parseInt %)))
@@ -161,6 +171,7 @@
   [sid tid max_query_cost max_computation_cost]
   {max_query_cost       MaxQueryCost
    max_computation_cost MaxComputationCost}
+  (api/check-403 (costs/enable-xrays))
   (fe/x-ray
    (fe/compare-features
     {:max-cost (max-cost max_query_cost max_computation_cost)}
@@ -173,6 +184,7 @@
   [sid tid field max_query_cost max_computation_cost]
   {max_query_cost       MaxQueryCost
    max_computation_cost MaxComputationCost}
+  (api/check-403 (costs/enable-xrays))
   (let [{:keys [comparison constituents]}
         (fe/x-ray
          (fe/compare-features

src/metabase/feature_extraction/core.clj

@@ -52,13 +52,13 @@
 
 (defn- sampled?
   [{:keys [max-cost] :as opts} dataset]
-  (and (costs/sample-only? max-cost)
+  (and (not (costs/full-scan? max-cost))
        (= (count (:rows dataset dataset)) max-sample-size)))
 
 (defn- extract-query-opts
   [{:keys [max-cost]}]
   (cond-> {}
-    (costs/sample-only? max-cost) (assoc :limit max-sample-size)))
+    (not (costs/full-scan? max-cost)) (assoc :limit max-sample-size)))
 
 (defmethod extract-features (type Field)
   [opts field]

src/metabase/feature_extraction/costs.clj

@@ -4,10 +4,21 @@
              [setting :refer [defsetting] :as setting]]
             [schema.core :as s]))
 
+(def ^:private query-costs {:cache     1
+                            :sample    2
+                            :full-scan 3
+                            :joins     4
+                            nil        3})
+
+(def ^:private computation-costs {:linear    1
+                                  :unbounded 2
+                                  :yolo      3
+                                  nil        2})
+
 (def MaxCost
   "Schema for `max-cost` parameter."
-  {:computation (s/enum :linear :unbounded :yolo)
-   :query       (s/enum :cache :sample :full-scan :joins)})
+  {:computation (apply s/enum (keys computation-costs))
+   :query       (s/enum (keys query-costs))})
 
 (def MaxCostBundles
   "Predefined `max-cost` bundles."
@@ -21,37 +32,64 @@
              (s/validate MaxCostBundles new-value)
              (setting/set-string! :xray-max-cost new-value)))
 
+(def ^:private max-cost-bundles {"exact"       {:query       :full-scan
+                                                :computation :unbounded}
+                                 "approximate" {:query       :sample
+                                                :computation :linear}
+                                 "extended"    {:query       :joins
+                                                :computation :unbounded}})
+
+(defn apply-global-cost-cap
+  "Cap given cost specification with `xray-max-cost`."
+  [max-cost]
+  (let [max-cost-cap (max-cost-bundles (xray-max-cost))]
+    {:query       (:query
+                   (if (> (-> max-cost :query query-costs)
+                          (-> max-cost-cap :query query-costs ))
+                     max-cost-cap
+                     max-cost))
+     :computation (:computation
+                   (if (> (-> max-cost :computation computation-costs)
+                          (-> max-cost-cap :computation computation-costs ))
+                     max-cost-cap
+                     max-cost))}))
+
 (defsetting enable-xrays
   "Should x-raying be available at all?"
   :type    :boolean
   :default true)
 
+(defn- min-cost
+  [costs min-cost]
+  (fn [cost]
+    (<= (costs min-cost) (costs cost))))
+
 (def ^{:arglists '([max-cost])} linear-computation?
   "Limit computation to O(n) or better."
-  (comp #{:linear} :computation))
+  (comp (min-cost computation-costs :linear) :computation))
 
 (def ^{:arglists '([max-cost])} unbounded-computation?
-  "Alow unbounded but always convergent computation.
+  "Allow unbounded but always convergent computation.
    Default if no cost limit is specified."
-  (comp (partial contains? #{:unbounded :yolo nil}) :computation))
+  (comp (min-cost computation-costs :unbounded) :computation))
 
 (def ^{:arglists '([max-cost])} yolo-computation?
-  "Alow any computation including full blown machine learning."
-  (comp #{:yolo} :computation))
+  "Allow any computation including full blown machine learning."
+  (comp (min-cost computation-costs :yolo) :computation))
 
 (def ^{:arglists '([max-cost])} cache-only?
   "Use cached data only."
-  (comp #{:cache} :query))
+  (comp (min-cost query-costs :cache) :query))
 
 (def ^{:arglists '([max-cost])} sample-only?
   "Only sample data."
-  (comp #{:sample} :query))
+  (comp (min-cost query-costs :sample) :query))
 
 (def ^{:arglists '([max-cost])} full-scan?
-  "Alow full table scans.
+  "Allow full table scans.
    Default if no cost limit is specified."
-  (comp (partial contains? #{:full-scan :joins nil}) :query))
+  (comp (min-cost query-costs :full-scan) :query))
 
-(def ^{:arglists '([max-cost])} alow-joins?
-  "Alow bringing in data from other tables if needed."
-  (comp #{:joins} :query))
+(def ^{:arglists '([max-cost])} allow-joins?
+  "Allow bringing in data from other tables if needed."
+  (comp (min-cost query-costs :joins) :query))

src/metabase/feature_extraction/feature_extractors.clj

@@ -365,7 +365,7 @@
                   (when (and resolution
                              (costs/unbounded-computation? max-cost))
                     (decompose-timeseries resolution series))}
-                 (when (and (costs/alow-joins? max-cost)
+                 (when (and (costs/allow-joins? max-cost)
                             (:aggregation query))
                    {:YoY (rolling-window-growth 365 query)
                     :MoM (rolling-window-growth 30 query)

test/metabase/feature_extraction/costs_test.clj

@@ -7,7 +7,7 @@
    true
    true
    true
-   false
+   true
    false
    true
    true
@@ -27,7 +27,21 @@
    (-> {:query :sample} sample-only? boolean)
    (-> {:query :full-scan} full-scan? boolean)
    (-> {:query :joins} full-scan? boolean)
-   (-> {:query :joins} alow-joins? boolean)
+   (-> {:query :joins} allow-joins? boolean)
    (-> nil full-scan? boolean)
-   (-> nil alow-joins? boolean)
+   (-> nil allow-joins? boolean)
    (-> {:query :sample} full-scan? boolean)])
+
+(expect
+  [true
+   true
+   false
+   false]
+  (let [xray-max-cost (constantly {:query       :full-scan
+                                   :computation :linear})
+        max-cost (apply-global-cost-cap {:computation :unbounded
+                                         :query       :sample})]
+    [(-> max-cost unbounded-computation?)
+     (-> max-cost linear-computation?)
+     (-> max-cost full-scan?)
+     (-> max-cost allow-joins? )]))