Fix integer overflows

project.clj

@@ -61,7 +61,7 @@
                  [environ "1.1.0"]                                    ; easy environment management
                  [hiccup "1.0.5"]                                     ; HTML templating
                  [honeysql "0.8.2"]                                   ; Transform Clojure data structures to SQL
-                 [kixi/stats "0.3.8"                                  ; Various statistic measures implemented as transducers
+                 [kixi/stats "0.3.9"                                  ; Various statistic measures implemented as transducers
                   :exclusions [org.clojure/test.check                 ; test.check and AVL trees are used in kixi.stats.random. Remove exlusion if using.
                                org.clojure/data.avl]]
                  [log4j/log4j "1.2.17"                                ; logging framework

src/metabase/feature_extraction/comparison.clj

@@ -28,7 +28,7 @@
               (fn [{:keys [magnitude-a magnitude-b product]}]
                 (some->> (fe/safe-divide product magnitude-a magnitude-b)
                          (- 1))))
-             (map vector a b)))
+             (map (comp (partial map double) vector) a b)))
 
 (defn head-tails-breaks
   "Pick out the cluster of N largest elements.
@@ -52,10 +52,12 @@
 (defmethod difference [Number Number]
   [a b]
   {:difference (cond
-                 (== a b 0)        0
+                 (== a b)          0
                  (zero? (max a b)) 1
-                 :else             (/ (- (max a b) (min a b))
-                                      (max (math/abs a) (math/abs b))))})
+                 :else             (let [a (double a)
+                                         b (double b)]
+                                     (/ (math/abs (- a b))
+                                        2 (max (math/abs a) (math/abs b)))))})
 
 (defmethod difference [Boolean Boolean]
   [a b]

src/metabase/feature_extraction/feature_extractors.clj

@@ -49,8 +49,14 @@
 (defn growth
   "Relative difference between `x1` an `x2`."
   [x2 x1]
-  (when (every? some? [x2 x1])
-    (safe-divide (* (if (neg? x1) -1 1) (- x2 x1)) x1)))
+  (when (and x1 x2)
+    (let [x2 (double x2)
+          x1 (double x1)]
+      (cond
+        (every? neg? [x1 x2])     (growth (- x1) (- x2))
+        (and (neg? x1) (pos? x2)) (- (growth x1 x2))
+        (nil? x1)                 nil
+        :else                     (/ (* (if (neg? x1) -1 1) (- x2 x1)) x1)))))
 
 (defn- merge-juxt
   [& fns]
@@ -192,11 +198,12 @@
    (redux/fuse (merge
                 {:histogram      h/histogram
                  :cardinality    cardinality
-                 :kurtosis       stats/kurtosis
-                 :skewness       stats/skewness
-                 :sum            (redux/with-xform + (remove nil?))
-                 :sum-of-squares (redux/with-xform + (comp (remove nil?)
-                                                           (map math/sq)))}
+                 :kurtosis       (redux/pre-step stats/kurtosis (stats/somef double))
+                 :skewness       (redux/pre-step stats/skewness (stats/somef double))
+                 :sum            (redux/with-xform +
+                                   (keep (stats/somef double)))
+                 :sum-of-squares (redux/with-xform +
+                                   (keep (stats/somef (comp math/sq double))))}
                 (when (isa? (:special_type field) :type/Category)
                   {:histogram-categorical h/histogram-categorical})))
    (merge-juxt
@@ -252,10 +259,14 @@
 (defmethod feature-extractor [Num Num]
   [_ field]
   (redux/post-complete
-   (redux/fuse {:linear-regression (stats/simple-linear-regression first second)
-                :correlation       (stats/correlation first second)
-                :covariance        (stats/covariance first second)})
-   (field-metadata-extractor field)))
+   (redux/pre-step
+    (redux/fuse {:linear-regression (stats/simple-linear-regression first second)
+                 :correlation       (stats/correlation first second)
+                 :covariance        (stats/covariance first second)})
+    (partial map (stats/somef double)))
+   (merge-juxt
+    (field-metadata-extractor field)
+    identity)))
 
 (def ^:private ^{:arglists '([t])} to-double
   "Coerce `DateTime` to `Double`."

test/metabase/feature_extraction/comparison_test.clj

@@ -20,8 +20,8 @@
    (cosine-distance [] [])])
 
 (expect
-  [0.5
-   0.0
+  [0.25
+   0
    1
    1
    0
@@ -60,6 +60,6 @@
    (#'c/flatten-map {:foo 4 :bar {:a 4 :b {:x 4 :y 7}}})])
 
 (expect
-  (approximately 0.5 0.1)
+  (approximately 0.3 0.1)
   (:distance (features-distance {:foo 2.0 :bar [1 2 3] :baz false}
                                 {:foo 12 :bar [10.7 0.2 3] :baz false})))

test/metabase/feature_extraction/feature_extractors_test.clj

@@ -16,16 +16,16 @@
    (safe-divide 4 0)])
 
 (expect
-  [(/ 23 100)
-   0.5
-   -1.0
+  [0.23
+   1.0
+   -0.5
    -5.0
-   1.2]
+   5.0]
   [(growth 123 100)
    (growth -0.1 -0.2)
    (growth -0.4 -0.2)
    (growth -0.4 0.1)
-   (growth 0.1 -0.5)])
+   (growth 0.1 -0.4)])
 
 (expect
   [{:foo 2