Add trend insight

frontend/src/metabase/xray/components/InsightCard.jsx

@@ -280,6 +280,22 @@ export class StationaryInsight extends Component {
     }
 }
 
+export class TrendInsight extends Component {
+    static insightType = "trend"
+    static title = "Trend"
+    static icon = "insight"
+
+    render() {
+	const { mode, shape } = this.props
+
+	return(
+		<InsightText>
+		Your data seems to be { mode } { shape }.
+		</InsightText>
+	)
+    }
+}
+
 /*
 export class RegimeChangeInsight extends Component {
     static insightType = "regime-change"
@@ -321,6 +337,7 @@ const INSIGHT_COMPONENTS = [
     SeasonalityInsight,
     StructuralBreaksInsight,
     StationaryInsight,
+    TrendInsight,
     // RegimeChangeInsight
 ]
 

src/metabase/feature_extraction/feature_extractors.clj

@@ -197,10 +197,10 @@
       :cardinality cardinality
       :kurtosis    (redux/pre-step stats/kurtosis (somef double))
       :skewness    (redux/pre-step stats/skewness (somef double))
-      :zeros       (redux/with-xform stats/count (filter (somef zero?)))}
+      :zeros       ((filter (somef zero?)) stats/count)}
      (when (costs/full-scan? max-cost)
-       {:sum            (redux/with-xform + (keep (somef double)))
-        :sum-of-squares (redux/with-xform + (keep (somef #(Math/pow % 2))))})
+       {:sum            ((keep (somef double)) +)
+        :sum-of-squares ((keep (somef #(Math/pow % 2))) +)})
      (when (isa? (:special_type field) :type/Category)
        {:histogram-categorical h/histogram-categorical})))
    (merge-juxt
@@ -313,20 +313,55 @@
   [{:keys [max-cost query]} field]
   (redux/post-complete
    (redux/pre-step
-    (redux/fuse {:linear-regression math/linear-regression
-                 :series            conj})
+    (redux/fuse {; y = a + b*x
+                 :linear-regression     (stats/simple-linear-regression
+                                         first second)
+                 ; y = e^a * x^b
+                 :power-law-regression  (stats/simple-linear-regression
+                                         #(Math/log (first %))
+                                         #(Math/log (second %)))
+                 ; y = a + b*ln(x)
+                 :log-linear-regression (stats/simple-linear-regression
+                                         #(Math/log (first %)) second)
+                 :series                conj})
     (fn [[^java.util.Date x y]]
       [(some-> x .getTime double) y]))
    (merge-juxt
     (field-metadata-extractor field)
-    (fn [{:keys [series linear-regression]}]
-      (let [resolution (infer-resolution query series)
+    (fn [{:keys [series linear-regression power-law-regression
+                 log-linear-regression] :as features}]
+      (let [best-fit   (transduce
+                        identity
+                        (redux/post-complete
+                         (redux/fuse
+                          {:linear-regression
+                           (let [[a b] linear-regression]
+                             (math/ssr (fn [x]
+                                         (+ a (* b x)))))
+                           :power-law-regression
+                           (let [[a b] power-law-regression]
+                             (math/ssr (fn [x]
+                                         (* (Math/exp a) (Math/pow x b)))))
+                           :log-linear-regression
+                           (let [[a b] log-linear-regression]
+                             (math/ssr (fn [x]
+                                         (+ a (* b (Math/log x))))))})
+                         (fn [fits]
+                           (println fits)
+                           (let [[model ssr] (apply min-key val fits)]
+                             {:model  model
+                              :ssr    ssr
+                              :params (features model)})))
+                        series)
+            resolution (infer-resolution query series)
             series     (if resolution
                          (ts/fill-timeseries resolution series)
                          series)]
         (merge {:resolution             resolution
                 :series                 series
-                :linear-regression      linear-regression
+                :linear-regression      (zipmap [:offset :slope]
+                                                linear-regression)
+                :best-fit               best-fit
                 :growth-series          (when resolution
                                           (->> series
                                                (partition 2 1)
@@ -341,7 +376,7 @@
                                                                  ts/period-length
                                                                  dec)
                                                          Long/MAX_VALUE)
-                                                    (/ (count series) 2))}
+                                                     (/ (count series) 2))}
                                       (map second series))}
                (when (and (costs/allow-joins? max-cost)
                           (:aggregation query))
@@ -353,7 +388,7 @@
 (defmethod comparison-vector [DateTime Num]
   [features]
   (-> features
-      (dissoc :resolution)
+      (dissoc :resolution :best-fit)
       ((get-method comparison-vector :default))))
 
 (defn- unpack-linear-regression
@@ -370,13 +405,14 @@
   [{:keys [field series] :as features}]
   (let [x-field (first field)]
     (-> features
-        (dissoc :series :autocorrelation)
+        (dissoc :series :autocorrelation :best-fit)
         (assoc :insights ((merge-juxt insights/noisiness
                                       insights/variation-trend
                                       insights/autocorrelation
                                       insights/seasonality
                                       insights/structural-breaks
-                                      insights/stationary)
+                                      insights/stationary
+                                      insights/trend)
                           features))
         (update :growth-series (partial series->dataset ts/from-double
                                         [x-field
@@ -524,7 +560,7 @@
   [opts field]
   (redux/post-complete
    (redux/fuse {:total-count stats/count
-                :nil-count   (redux/with-xform stats/count (filter nil?))})
+                :nil-count   ((filter nil?) stats/count)})
    (merge-juxt
     (field-metadata-extractor field)
     (fn [{:keys [total-count nil-count]}]

src/metabase/feature_extraction/insights.clj

@@ -209,3 +209,20 @@
                                    (* n (- n 1)))))]
                       (math/significant? t (d/t-distribution k) (/ 0.05 2)))))))
          (every? false?))))
+
+(definsight trend
+  "What is the best (simple) trend model?
+
+   We try fitting a linear, power law, and logarithmic models and pick the one
+   with the smallest sum of residual squares.
+   https://en.wikipedia.org/wiki/Residual_sum_of_squares
+   https://en.wikipedia.org/wiki/Simple_linear_regression
+   http://mathworld.wolfram.com/LeastSquaresFittingPowerLaw.html
+   http://mathworld.wolfram.com/LeastSquaresFittingLogarithmic.html"
+  [best-fit]
+  {:mode  (if (-> best-fit :params second pos?)
+            :growing
+            :decreasing)
+   :shape ({:linear-regression     :linearly
+            :power-law-regression  :exponentally
+            :log-linear-regression :logarithmically} (:model best-fit))})

src/metabase/feature_extraction/math.clj

@@ -59,9 +59,7 @@
   ([{:keys [lag max-lag]} xs]
    {:pre [(or lag max-lag)]}
    (if lag
-     (transduce identity (redux/post-complete
-                          (stats/correlation first second)
-                          (stats/somef #(max (min % 1.0) -1.0)))
+     (transduce identity (stats/correlation first second)
                 (map vector xs (drop lag xs)))
      (let [n (count xs)]
        (reduce (fn [best lag]
@@ -69,19 +67,19 @@
                    (if (and (some-> r
                                     (* (math/sqrt (- n lag)))
                                     (significant? (d/normal 0 1) (/ 0.05 2)))
-                            (pos? (compare [(math/abs r) (- lag)]
-                                           [(math/abs (:autocorrelation best 0))
-                                            (- (:lag best 0))])))
+                            (> (math/abs r) (math/abs (:autocorrelation best 0))))
                      {:autocorrelation r
                       :lag             lag}
                      best)))
                nil
                (range 1 (inc max-lag)))))))
 
-(def linear-regression
-  "Transducer that calculates (simple) linear regression."
-  (redux/post-complete (stats/simple-linear-regression first second)
-                       (partial zipmap [:offset :slope])))
+(defn ssr
+  "Transducer that calculates residual sum of squares.
+   https://en.wikipedia.org/wiki/Residual_sum_of_squares"
+  [model]
+  (redux/pre-step + (fn [[x y]]
+                      (math/sq (- y (model x))))))
 
 (def magnitude
   "Transducer that claclulates magnitude (Euclidean norm) of given vector.