MySQL tweaks

src/metabase/driver/generic_sql/query_processor.clj

@@ -90,11 +90,6 @@
 
 (defmethod apply-form :default [form]) ;; nothing
 
-(defn- cast-as-date
-  "Generate a korma form to cast FIELD-OR-VALUE to a `DATE`."
-  [field-or-value]
-  (utils/func "CAST(%s AS DATE)" [field-or-value]))
-
 (defprotocol IGenericSQLFormattable
   (formatted [this] [this include-as?]))
 
@@ -116,7 +111,7 @@
     ([this]
      (formatted this false))
     ([{unit :unit, {:keys [field-name base-type special-type], :as field} :field} include-as?]
-     (let [field (cast-as-date (formatted field))]
+     (let [field (i/date (:driver *query*) unit (formatted field))]
        (if include-as? [field (keyword field-name)]
            field))))
 
@@ -147,9 +142,9 @@
   (formatted
     ([this]
      (formatted this false))
-    ([{:keys [value]} _]
+    ([{value :value, {unit :unit} :field} _]
      ;; prevent Clojure from converting this to #inst literal, which is a util.date
-     (cast-as-date `(Timestamp/valueOf ~(.toString value))))))
+     (i/date (:driver *query*) unit `(Timestamp/valueOf ~(.toString value))))))
 
 
 (defmethod apply-form :aggregation [[_ {:keys [aggregation-type field]}]]

src/metabase/driver/mysql.clj

@@ -73,7 +73,7 @@
   (utils/func (case seconds-or-milliseconds
                 :seconds      "FROM_UNIXTIME(%s)"
                 :milliseconds "FROM_UNIXTIME(%s / 1000)")
-              [field-or-value]))
+                [field-or-value]))
 
 (defn- timezone->set-timezone-sql [_ timezone]
   ;; If this fails you need to load the timezone definitions from your system into MySQL;
@@ -90,19 +90,21 @@
 ;; Truncating to a quarter is trickier since there aren't any format strings.
 ;; See the explanation in the H2 driver, which does the same thing but with slightly different syntax.
 (defn- trunc-to-quarter [field-or-value]
-  (funcs "STR_TO_DATE(%s, '%%Y%%m')"
+  (funcs "STR_TO_DATE(%s, '%%Y-%%m-%%d')"
          ["CONCAT(%s)"
           ["YEAR(%s)" field-or-value]
-          ["((QUARTER(%s) * 3) - 2)" field-or-value]]))
+          (k/raw "'-'")
+          ["((QUARTER(%s) * 3) - 2)" field-or-value]
+          (k/raw "'-01'")]))
 
 (defn- date [_ unit field-or-value]
   (if (= unit :quarter)
     (trunc-to-quarter field-or-value)
     (utils/func (case unit
                   :default         "TIMESTAMP(%s)"
-                  :minute          (trunc-with-format "%Y%m%d%H%i")
+                  :minute          (trunc-with-format "%Y-%m-%d %H:%i")
                   :minute-of-hour  "MINUTE(%s)"
-                  :hour            (trunc-with-format "%Y%m%d%H")
+                  :hour            (trunc-with-format "%Y-%m-%d %H")
                   :hour-of-day     "HOUR(%s)"
                   :day             "DATE(%s)"
                   :day-of-week     "DAYOFWEEK(%s)"
@@ -110,9 +112,10 @@
                   :day-of-year     "DAYOFYEAR(%s)"
                   ;; To convert a YEARWEEK (e.g. 201530) back to a date you need tell MySQL which day of the week to use,
                   ;; because otherwise as far as MySQL is concerned you could be talking about any of the days in that week
-                  :week            "STR_TO_DATE(CONCAT(YEARWEEK(%s), 'Sunday'), '%%X%%V%%W')"
-                  :week-of-year    "WEEK(%s)"
-                  :month           (trunc-with-format "%Y%m")
+                  :week            "STR_TO_DATE(CONCAT(YEARWEEK(%s), ' Sunday'), '%%X%%V %%W')"
+                  ;; mode 6: Sunday is first day of week, first week of year is the first one with 4+ days
+                  :week-of-year    "(WEEK(%s, 6) + 1)"
+                  :month           "STR_TO_DATE(CONCAT(DATE_FORMAT(%s, '%%Y-%%m'), '-01'), '%%Y-%%m-%%d')"
                   :month-of-year   "MONTH(%s)"
                   :quarter-of-year "QUARTER(%s)"
                   :year            "YEAR(%s)")

src/metabase/driver/postgres.clj

@@ -123,21 +123,23 @@
 (defn- date [_ unit field-or-value]
   (utils/func (case unit
                 :default         "CAST(%s AS TIMESTAMP)"
-                :minute          "DATE_TRUNC('minute', %s)" ; or CAST as timestamp?
-                :minute-of-hour  "EXTRACT(MINUTE FROM %s)"
+                :minute          "DATE_TRUNC('minute', %s)"
+                :minute-of-hour  "CAST(EXTRACT(MINUTE FROM %s) AS INTEGER)"
                 :hour            "DATE_TRUNC('hour', %s)"
-                :hour-of-day     "EXTRACT(HOUR FROM %s)"
+                :hour-of-day     "CAST(EXTRACT(HOUR FROM %s) AS INTEGER)"
                 :day             "CAST(%s AS DATE)"
-                :day-of-week     "(EXTRACT(DOW FROM %s) + 1)" ; Postgres DOW is 0 (Sun) - 6 (Sat); increment this to be consistent with Java, H2, MySQL, and Mongo (1-7)
-                :day-of-month    "EXTRACT(DAY FROM %s)"
-                :day-of-year     "EXTRACT(DOY FROM %s)"
-                :week            "DATE_TRUNC('week', %s)"
-                :week-of-year    "EXTRACT(WEEK FROM %s)"
+                ;; Postgres DOW is 0 (Sun) - 6 (Sat); increment this to be consistent with Java, H2, MySQL, and Mongo (1-7)
+                :day-of-week     "(CAST(EXTRACT(DOW FROM %s) AS INTEGER) + 1)"
+                :day-of-month    "CAST(EXTRACT(DAY FROM %s) AS INTEGER)"
+                :day-of-year     "CAST(EXTRACT(DOY FROM %s) AS INTEGER)"
+                ;; Postgres weeks start on Monday, so shift this date into the proper bucket and then decrement the resulting day
+                :week            "(DATE_TRUNC('week', (%s + INTERVAL '1 day')) - INTERVAL '1 day')"
+                :week-of-year    "CAST(EXTRACT(WEEK FROM (%s + INTERVAL '1 day')) AS INTEGER)"
                 :month           "DATE_TRUNC('month', %s)"
-                :month-of-year   "EXTRACT(MONTH FROM %s)"
+                :month-of-year   "CAST(EXTRACT(MONTH FROM %s) AS INTEGER)"
                 :quarter         "DATE_TRUNC('quarter', %s)"
-                :quarter-of-year "EXTRACT(QUARTER FROM %s)"
-                :year            "DATE_TRUNC('year', %s)")
+                :quarter-of-year "CAST(EXTRACT(QUARTER FROM %s) AS INTEGER)"
+                :year            "CAST(EXTRACT(YEAR FROM %s) AS INTEGER)")
               [field-or-value]))
 
 (defn- date-interval [_ unit amount]

src/metabase/driver/query_processor.clj

@@ -108,27 +108,6 @@
                                                                                                                    :direction :ascending}))))))))
 
 
-(defn- post-convert-unix-timestamps-to-dates
-  "Convert the values of Unix timestamps (for `Fields` whose `:special_type` is `:timestamp_seconds` or `:timestamp_milliseconds`) to dates."
-  [qp]
-  (fn [query]
-    (let [{:keys [cols rows], :as results} (qp query)
-          timestamp-seconds-col-indecies   (u/indecies-satisfying #(= (:special_type %) :timestamp_seconds)      cols)
-          timestamp-millis-col-indecies    (u/indecies-satisfying #(= (:special_type %) :timestamp_milliseconds) cols)]
-      (if-not (or (seq timestamp-seconds-col-indecies)
-                  (seq timestamp-millis-col-indecies))
-        ;; If we don't have any columns whose special type is a seconds or milliseconds timestamp return results as-is
-        results
-        ;; Otherwise go modify the results of each row
-        (update-in results [:rows] #(for [row %]
-                                      (for [[i val] (m/indexed row)]
-                                        (cond
-                                          (instance? java.util.Date val)               val ; already converted to Date as part of preprocessing,
-                                          (contains? timestamp-seconds-col-indecies i) (java.sql.Date. (* val 1000)) ; nothing to do here
-                                          (contains? timestamp-millis-col-indecies i)  (java.sql.Date. val)
-                                          :else                                        val))))))))
-
-
 (defn- pre-cumulative-sum
   "Rewrite queries containing a cumulative sum (`cum_sum`) aggregation to simply fetch the values of the aggregate field instead.
    (Cumulative sum is a special case; it is implemented in post-processing).
@@ -270,7 +249,6 @@
           post-add-row-count-and-status
           pre-add-implicit-fields
           pre-add-implicit-breakout-order-by
-          post-convert-unix-timestamps-to-dates
           cumulative-sum
           limit
           annotate/post-annotate
@@ -284,7 +262,6 @@
     ((<<- wrap-catch-exceptions
           driver-wrap-process-query
           post-add-row-count-and-status
-          post-convert-unix-timestamps-to-dates
           limit
           wrap-guard-multiple-calls
           driver-process-query) query)))

src/metabase/driver/query_processor/expand.clj

@@ -76,8 +76,8 @@
      (do (assert-driver-supports :foreign-keys)
          (map->FieldPlaceholder {:field-id dest-field-id, :fk-field-id fk-field-id}))
 
-     ["datetime_field" field-id (unit :guard datetime-field-unit?)]
-     (assoc (ph field-id)
+     ["datetime_field" id (unit :guard datetime-field-unit?)]
+     (assoc (ph id)
             :datetime-unit (keyword unit))
 
      _ (throw (Exception. (str "Invalid field: " field-id))))))

src/metabase/driver/query_processor/interface.clj

@@ -56,7 +56,7 @@
 (def ^:const datetime-field-units
   "Valid units for a `DateTimeField`."
   #{:default :minute :minute-of-hour :hour :hour-of-day :day :day-of-week :day-of-month :day-of-year
-    :week :week-of-year :month :month-of-year :quarter :quarter-of-year})
+    :week :week-of-year :month :month-of-year :quarter :quarter-of-year :year})
 
 (defn datetime-field-unit? [unit]
   (contains? datetime-field-units (keyword unit)))

test/metabase/driver/query_processor_test.clj

@@ -1074,3 +1074,172 @@
  (Q return first-row
     aggregate count of venues
     filter != price 1 2))
+
+
+;; +-------------------------------------------------------------------------------------------------------------+
+;; |                                                NEW DATE STUFF                                               |
+;; +-------------------------------------------------------------------------------------------------------------+
+
+
+;; ## BUCKETING
+
+(defn- sad-toucan-incidents-with-bucketing [unit]
+  (vec (Q dataset sad-toucan-incidents
+          aggregate count of incidents
+          breakout ["datetime_field" (id :incidents :timestamp) unit]
+          limit 10
+          return rows)))
+
+(datasets/expect-with-datasets #{:h2 :postgres :mysql}
+  [[#inst "2015-06-01T10:31" 1]
+   [#inst "2015-06-01T16:06" 1]
+   [#inst "2015-06-01T17:23" 1]
+   [#inst "2015-06-01T18:55" 1]
+   [#inst "2015-06-01T21:04" 1]
+   [#inst "2015-06-01T21:19" 1]
+   [#inst "2015-06-02T02:13" 1]
+   [#inst "2015-06-02T05:37" 1]
+   [#inst "2015-06-02T08:20" 1]
+   [#inst "2015-06-02T11:11" 1]]
+  (sad-toucan-incidents-with-bucketing :default))
+
+(datasets/expect-with-datasets #{:h2 :postgres :mysql}
+  [[#inst "2015-06-01T10:31" 1]
+   [#inst "2015-06-01T16:06" 1]
+   [#inst "2015-06-01T17:23" 1]
+   [#inst "2015-06-01T18:55" 1]
+   [#inst "2015-06-01T21:04" 1]
+   [#inst "2015-06-01T21:19" 1]
+   [#inst "2015-06-02T02:13" 1]
+   [#inst "2015-06-02T05:37" 1]
+   [#inst "2015-06-02T08:20" 1]
+   [#inst "2015-06-02T11:11" 1]]
+  (sad-toucan-incidents-with-bucketing :minute))
+
+(datasets/expect-with-datasets #{:h2 :postgres :mysql}
+  [[0 5]
+   [1 4]
+   [2 2]
+   [3 4]
+   [4 4]
+   [5 3]
+   [6 5]
+   [7 1]
+   [8 1]
+   [9 1]]
+  (sad-toucan-incidents-with-bucketing :minute-of-hour))
+
+(datasets/expect-with-datasets #{:h2 :postgres :mysql}
+  [[#inst "2015-06-01T10" 1]
+   [#inst "2015-06-01T16" 1]
+   [#inst "2015-06-01T17" 1]
+   [#inst "2015-06-01T18" 1]
+   [#inst "2015-06-01T21" 2]
+   [#inst "2015-06-02T02" 1]
+   [#inst "2015-06-02T05" 1]
+   [#inst "2015-06-02T08" 1]
+   [#inst "2015-06-02T11" 1]
+   [#inst "2015-06-02T13" 1]]
+  (sad-toucan-incidents-with-bucketing :hour))
+
+(datasets/expect-with-datasets #{:h2 :postgres :mysql}
+  [[0 8]
+   [1 9]
+   [2 7]
+   [3 10]
+   [4 10]
+   [5 9]
+   [6 6]
+   [7 5]
+   [8 7]
+   [9 7]]
+  (sad-toucan-incidents-with-bucketing :hour-of-day))
+
+(datasets/expect-with-datasets #{:h2 :postgres :mysql}
+  [[#inst "2015-06-01T07" 8]
+   [#inst "2015-06-02T07" 9]
+   [#inst "2015-06-03T07" 9]
+   [#inst "2015-06-04T07" 4]
+   [#inst "2015-06-05T07" 11]
+   [#inst "2015-06-06T07" 8]
+   [#inst "2015-06-07T07" 6]
+   [#inst "2015-06-08T07" 10]
+   [#inst "2015-06-09T07" 6]
+   [#inst "2015-06-10T07" 10]]
+  (sad-toucan-incidents-with-bucketing :day))
+
+(datasets/expect-with-datasets #{:h2 :postgres :mysql}
+  [[1 29]
+   [2 36]
+   [3 33]
+   [4 29]
+   [5 13]
+   [6 38]
+   [7 22]]
+  (sad-toucan-incidents-with-bucketing :day-of-week))
+
+(datasets/expect-with-datasets #{:h2 :postgres :mysql}
+  [[1 8]
+   [2 9]
+   [3 9]
+   [4 4]
+   [5 11]
+   [6 8]
+   [7 6]
+   [8 10]
+   [9 6]
+   [10 10]]
+  (sad-toucan-incidents-with-bucketing :day-of-month))
+
+(datasets/expect-with-datasets #{:h2 :postgres :mysql}
+  [[152 8]
+   [153 9]
+   [154 9]
+   [155 4]
+   [156 11]
+   [157 8]
+   [158 6]
+   [159 10]
+   [160 6]
+   [161 10]]
+  (sad-toucan-incidents-with-bucketing :day-of-year))
+
+(datasets/expect-with-datasets #{:h2 :postgres :mysql}
+  [[#inst "2015-05-31T07" 49]
+   [#inst "2015-06-07T07" 47]
+   [#inst "2015-06-14T07" 39]
+   [#inst "2015-06-21T07" 58]
+   [#inst "2015-06-28T07" 7]]
+  (sad-toucan-incidents-with-bucketing :week))
+
+(datasets/expect-with-datasets #{:h2 :postgres :mysql}
+  [[23 49]
+   [24 47]
+   [25 39]
+   [26 58]
+   [27 7]]
+  (sad-toucan-incidents-with-bucketing :week-of-year))
+
+(datasets/expect-with-datasets #{:h2 :postgres :mysql}
+  [[#inst "2015-06-01T07" 200]]
+  (sad-toucan-incidents-with-bucketing :month))
+
+(datasets/expect-with-datasets #{:h2 :postgres :mysql}
+  [[6 200]]
+  (sad-toucan-incidents-with-bucketing :month-of-year))
+
+(datasets/expect-with-datasets #{:h2 :postgres :mysql}
+  [[#inst "2015-04-01T07" 200]]
+  (sad-toucan-incidents-with-bucketing :quarter))
+
+(datasets/expect-with-datasets #{:h2 :postgres :mysql}
+  [[2 200]]
+  (sad-toucan-incidents-with-bucketing :quarter-of-year))
+
+(datasets/expect-with-datasets #{:h2 :postgres :mysql}
+  [[2015 200]]
+  (sad-toucan-incidents-with-bucketing :year))
+
+;; RELATIVE DATES
+
+;; SYNTACTIC SUGAR

test/metabase/test/util/q.clj

@@ -38,25 +38,25 @@
 (def ^:dynamic *table-name* nil)
 
 (defmacro field [f]
-  {:pre [(symbol? f)]}
   (core/or
-   (let [f (name f)]
-     (u/cond-let
-      ;; x->y <-> ["fk->" x y]
-      [[_ from to] (re-matches #"^(.+)->(.+)$" f)]
-      ["fk->" `(field ~(symbol from)) `(field ~(symbol to))]
-
-      ;; x...y <-> ?
-      [[_ f sub] (re-matches #"^(.+)\.\.\.(.+)$" f)]
-      `(~@(macroexpand-1 `(field ~(symbol f))) ~(keyword sub))
-
-      ;; ag.0 <-> ["aggregation" 0]
-      [[_ ag-field-index] (re-matches #"^ag\.(\d+)$" f)]
-      ["aggregation" (Integer/parseInt ag-field-index)]
-
-      ;; table.field <-> (id table field)
-      [[_ table field] (re-matches #"^([^\.]+)\.([^\.]+)$" f)]
-      `(data/id ~(keyword table) ~(keyword field))))
+   (if-not (symbol? f) f
+     (let [f (name f)]
+       (u/cond-let
+        ;; x->y <-> ["fk->" x y]
+        [[_ from to] (re-matches #"^(.+)->(.+)$" f)]
+        ["fk->" `(field ~(symbol from)) `(field ~(symbol to))]
+
+        ;; x...y <-> ?
+        [[_ f sub] (re-matches #"^(.+)\.\.\.(.+)$" f)]
+        `(~@(macroexpand-1 `(field ~(symbol f))) ~(keyword sub))
+
+        ;; ag.0 <-> ["aggregation" 0]
+        [[_ ag-field-index] (re-matches #"^ag\.(\d+)$" f)]
+        ["aggregation" (Integer/parseInt ag-field-index)]
+
+        ;; table.field <-> (id table field)
+        [[_ table field] (re-matches #"^([^\.]+)\.([^\.]+)$" f)]
+        `(data/id ~(keyword table) ~(keyword field)))))
 
    ;; fallback : (id *table-name* field)
    `(data/id *table-name* ~(keyword f))))