diff --git a/sample_dataset/metabase/sample_dataset/generate.clj b/sample_dataset/metabase/sample_dataset/generate.clj index fdeec5267a26201492d73b20ca4b1a835e1945a6..d51dd122abcb8ec9472e1af37607a1f90efc8246 100644 --- a/sample_dataset/metabase/sample_dataset/generate.clj +++ b/sample_dataset/metabase/sample_dataset/generate.clj @@ -295,7 +295,7 @@ (let [n (count xs)] (map-indexed (fn [i x] ; Limit the noise to [0.1, 2.1]. - (update x k * (+ 1 (* (/ i n) (- (rand 2) 0.9))))) + (update x k * (+ 1 (* (/ i n) (- (* 2 (rand)) 0.9))))) xs))) (defn- add-seasonality diff --git a/src/metabase/feature_extraction/insights.clj b/src/metabase/feature_extraction/insights.clj index 3271b223f7fe999464ed8b54ac1c3c79ead2906b..b2c2d35bfdba2f0d9554475531b55316a5f00309 100644 --- a/src/metabase/feature_extraction/insights.clj +++ b/src/metabase/feature_extraction/insights.clj @@ -83,7 +83,7 @@ [resolution series] (when resolution (transduce - (comp (x/partition (ts/period-length resolution) 1 + (comp (x/partition (ts/period-length resolution) (comp (map second) (x/reduce h/histogram))) (map-indexed (fn [idx histogram] @@ -104,7 +104,7 @@ (- (* n s-xx) (sq s-x)))) (- n 2) (- (* n s-xx) (sq s-x)))] - (when (and (not= slope 0) + (when (and (not= slope 0.0) (math/significant? (/ slope slope-error) (d/t-distribution (- n 2)) (/ 0.05 2))) @@ -183,8 +183,8 @@ https://en.wikipedia.org/wiki/Stationary_process" [series resolution] (when-let [n (ts/period-length resolution)] - (transduce (comp (x/partition n 1 (comp (map second) - (x/reduce h/histogram))) + (transduce (comp (x/partition n (comp (map second) + (x/reduce h/histogram))) (map (fn [histogram] {:mean (h.impl/mean histogram) :var (h.impl/variance histogram)})) @@ -193,7 +193,7 @@ ([] true) ([acc] acc) ([_ [{mean1 :mean var1 :var} {mean2 :mean var2 :var}]] - (if (= var1 var2 0) + (if (= var1 var2 0.0) true (let [t (/ (- mean1 mean2) (num/sqrt (/ (+ var1 var2) n))) diff --git a/test/metabase/feature_extraction/insights_test.clj b/test/metabase/feature_extraction/insights_test.clj new file mode 100644 index 0000000000000000000000000000000000000000..3a47a4db900abff4dc2d4ac0c4d4737d9df9e35d --- /dev/null +++ b/test/metabase/feature_extraction/insights_test.clj @@ -0,0 +1,32 @@ +(ns metabase.feature-extraction.insights-test + (:require [expectations :refer :all] + [medley.core :as m] + [metabase.feature-extraction.insights :refer :all])) + +(expect + [true + nil] + (map :stationary [(stationary {:series (m/indexed (repeat 100 1)) + :resolution :month}) + (stationary {:series (m/indexed (range 100)) + :resolution :month})])) + +(expect + [{:mode :increasing} + {:mode :increasing} + nil] + (map :variation-trend + [(variation-trend {:series (map-indexed + (fn [i x] + [i (* x (+ 1 (* (/ i 100) + (- (* 2 (rand)) 0.9))))]) + (repeatedly 100 #(rand-int 10))) + :resolution :month}) + (variation-trend {:series (map-indexed + (fn [i x] + [i (* x (+ 1 (* (/ (- 100 i) 100) + (- (* 2 (rand)) 0.9))))]) + (repeatedly 100 rand)) + :resolution :month}) + (variation-trend {:series (m/indexed (repeat 100 1)) + :resolution :month})]))