Merge pull request #6287 from metabase/fix-fill-timeseries

Make fill-timeseries work on not evenly spaced data

src/metabase/feature_extraction/feature_extractors.clj

@@ -292,18 +292,26 @@
   (somef (comp t.coerce/from-long long)))
 
 (defn- fill-timeseries
-  "Given a coll of `[DateTime, Any]` pairs evenly spaced `step` apart, fill
+  "Given a coll of `[DateTime, Num]` pairs evenly spaced `step` apart, fill
    missing points with 0."
-  [step ts]
-  (let [ts-index (into {} ts)]
+  [resolution ts]
+  (let [[step rounder] (case resolution
+                         :month   [(t/months 1) t/month]
+                         :quarter [(t/months 3) t/month]
+                         :year    [(t/years 1) t/year]
+                         :week    [(t/weeks 1) t/day]
+                         :day     [(t/days 1) t/day]
+                         :hour    [(t/hours 1) t/day]
+                         :minute  [(t/minutes 1) t/minute])
+        ts             (for [[x y] ts]
+                         [(-> x from-double (t/floor rounder)) y])
+        ts-index       (into {} ts)]
     (into []
-      (comp (map to-double)
-            (take-while (partial >= (-> ts last first)))
+      (comp (take-while (partial (complement t/before?) (-> ts last first)))
             (map (fn [t]
-                   [t (ts-index t 0)])))
+                   [(to-double t) (ts-index t 0)])))
       (some-> ts
               ffirst
-              from-double
               (t.periodic/periodic-seq step)))))
 
 (defn- decompose-timeseries
@@ -390,15 +398,7 @@
     (fn [{:keys [series linear-regression]}]
       (let [resolution (infer-resolution query series)
             series     (if resolution
-                         (fill-timeseries (case resolution
-                                            :month   (t/months 1)
-                                            :quarter (t/months 3)
-                                            :year    (t/years 1)
-                                            :week    (t/weeks 1)
-                                            :day     (t/days 1)
-                                            :hour    (t/hours 1)
-                                            :minute  (t/minutes 1))
-                                          series)
+                         (fill-timeseries resolution series)
                          series)]
         (merge {:resolution             resolution
                 :series                 series

test/metabase/feature_extraction/async_test.clj

@@ -10,12 +10,12 @@
     (done? (ComputationJob job-id))))
 
 (expect
-  [true false false]
-  (let [job-id (compute (gensym) #(loop [] (Thread/sleep 100) (recur)))]
+  [true :canceled false]
+  (let [job-id (compute (gensym) #(loop [] (Thread/sleep 100) (recur)))
+        r? (running? (ComputationJob job-id))]
     (Thread/sleep 100)
-    (let [r? (running? (ComputationJob job-id))]
-      (cancel (ComputationJob job-id))
-      [r? (done? (ComputationJob job-id)) (running? (ComputationJob job-id))])))
+    (cancel (ComputationJob job-id))
+    [r? (:status (ComputationJob job-id)) (running? (ComputationJob job-id))]))
 
 (expect
   {:status :done

test/metabase/feature_extraction/feature_extractors_test.clj

@@ -78,9 +78,9 @@
    [(make-timestamp 2016 11) 0]
    [(make-timestamp 2016 12) 0]
    [(make-timestamp 2017 1) 25]]
-  (#'fe/fill-timeseries (t/months 1) [[(make-timestamp 2016 1) 12]
-                                     [(make-timestamp 2016 3) 4]
-                                      [(make-timestamp 2017 1) 25]]))
+  (#'fe/fill-timeseries :month [[(make-timestamp 2016 1 12 4) 12]
+                                [(make-timestamp 2016 3 2 2) 4]
+                                [(make-timestamp 2017 1) 25]]))
 
 (expect
   [2

test/metabase/test/async.clj

@@ -9,10 +9,13 @@
   "Retries fetching job results until `max-retries` times."
   [user job-id]
   (loop [tries 1]
-    (let [{:keys [status result]} ((user->client user) :get 200 (str "async/" job-id))]
+    (let [{:keys [status result] :as response}
+          ((user->client user) :get 200 (str "async/" job-id))]
       (cond
-        (= "done" status)     result
-        (> tries max-retries) (throw (ex-info "Timeout. Max retries exceeded."))
+        (= status "done")     result
+        (= status "error")    (throw (ex-info (str "Error encountered.\n" result)
+                                       result))
+        (> tries max-retries) (throw (ex-info "Timeout. Max retries exceeded." {}))
         :else                 (do
                                 (log/info (format "Waiting for computation to finish. Retry: %" tries))
                                 (Thread/sleep (* 100 tries))