Add multimodality insight

frontend/src/metabase/xray/components/InsightCard.jsx

@@ -72,7 +72,7 @@ export class NilsInsight extends Component {
 
 export class ZerosInsight extends Component {
     static insightType = "zeros"
-    static title = "0s in your data"
+    static title = "Zeros in your data"
     static icon = "warning"
 
     render() {
@@ -181,6 +181,25 @@ export class SeasonalityInsight extends Component {
         )
     }
 }
+
+const multimodalDefinition = "Data distribution with multiple peaks (modes)."
+const multimodalLink = "https://en.wikipedia.org/wiki/Multimodal_distribution"
+
+export class MultimodalInsight extends Component {
+    static insightType = "multimodal"
+    static title = "Multimodal"
+    static icon = "warning"
+
+    render() {
+        return (
+            <InsightText>
+                Your data looks to be <TermWithDefinition definition={multimodalDefinition} link={multimodalLink}>
+                    multimodal
+                </TermWithDefinition>. This is often the case when different segments of data are mixed together.
+            </InsightText>
+        )
+    }
+}
 /*
 export class RegimeChangeInsight extends Component {
     static insightType = "regime-change"
@@ -217,6 +236,7 @@ const INSIGHT_COMPONENTS = [
     // numeric fields
     NormalRangeInsight,
     ZerosInsight,
+    MultimodalInsight,
     // timeseries
     NoisinessInsight,
     VariationTrendInsight,

project.clj

@@ -77,6 +77,8 @@
                  [medley "0.8.4"]                                     ; lightweight lib of useful functions
                  [metabase/throttle "1.0.1"]                          ; Tools for throttling access to API endpoints and other code pathways
                  [mysql/mysql-connector-java "5.1.39"]                ;  !!! Don't upgrade to 6.0+ yet -- that's Java 8 only !!!
+                 [net.sourceforge.jdistlib/jdistlib "0.4.5"           ; Distribution statistic tests
+                  :exclusions [com.github.wendykierp/JTransforms]]
                  [net.sf.cssbox/cssbox "4.12"                         ; HTML / CSS rendering
                   :exclusions [org.slf4j/slf4j-api]]
                  [com.clearspring.analytics/stream "2.9.5"            ; Various sketching algorithms

sample_dataset/metabase/sample_dataset/generate.clj

@@ -67,11 +67,20 @@
 (defn- random-company-name []
   (first (company/names)))
 
+(defn- rejection-sample
+  [min max dist]
+  (let [x (dist/sample dist)]
+    (if (<= min x max)
+      x
+      (rejection-sample min max dist))))
+
 (defn- random-price [min max]
-  (let [range (- max min)]
-    (-> (rand-int (* range 100))
-        (/ 100.0)
-        (+ min))))
+  (let [range    (- max min)
+        m1       (+ min (* range 0.3))
+        m2       (+ min (* range 0.75))
+        variance (/ range 8)]
+    (rejection-sample min max (rand-nth [(dist/normal m1 variance)
+                                         (dist/normal m2 variance)]))))
 
 (def ^:private ^:const product-names
   {:adjective '[Small, Ergonomic, Rustic, Intelligent, Gorgeous, Incredible, Fantastic, Practical, Sleek, Awesome, Enormous, Mediocre, Synergistic, Heavy-Duty, Lightweight, Aerodynamic, Durable]
@@ -267,7 +276,7 @@
   [k xs]
   (let [n (count xs)]
     (map-indexed (fn [i x]
-                   (update x k * (/ i n) (rand)))
+                   (update x k * (+ 1 (* (/ i n) (- (rand 2) 0.9)))))
                  xs)))
 
 (defn add-seasonality

src/metabase/feature_extraction/feature_extractors.clj

@@ -259,7 +259,8 @@
       (update :histogram (partial histogram->dataset field))
       (assoc :insights ((merge-juxt insights/normal-range
                                     insights/zeros
-                                    insights/nils)
+                                    insights/nils
+                                    insights/multimodal)
                         features))
       (dissoc :has-nils? :var>sd? :0<=x<=1? :-1<=x<=1? :all-distinct?
               :positive-definite? :var>sd? :uniqueness :min-vs-max)))

src/metabase/feature_extraction/insights.clj

 (ns metabase.feature-extraction.insights
   "Data insights -- morsels of prepackaged analysis."
-  (:require [clojure.math.numeric-tower :as num]
+  (:require [bigml.histogram.core :as h.impl]
+            [clojure.math.numeric-tower :as num]
             [distributions.core :as d]
             [kixi.stats.core :as stats]
             [redux.core :as redux]
             [metabase.feature-extraction
              [histogram :as h]
              [math :as math]
-             [timeseries :as ts]]))
+             [timeseries :as ts]])
+  (:import net.sourceforge.jdistlib.disttest.DistributionTest)
+  )
 
 (defmacro ^:private definsight
   [insight docs features & body]
@@ -130,3 +133,15 @@
         {:quality (if (< diff 1)
                     :weak
                     :strong)}))))
+
+(definsight multimodal
+  "Is the data multimodal?
+   https://en.wikipedia.org/wiki/Multimodal_distribution
+   http://www.nicprice.net/diptest/Hartigan_1985_AnnalStat.pdf"
+  [histogram]
+  (-> histogram
+      (h.impl/sample 1000)
+      double-array
+      DistributionTest/diptest
+      second
+      (< 0.05)))