From d0be9fc4c291438204e792f3981bf7fe5a42878a Mon Sep 17 00:00:00 2001
From: Ngoc Khuat <qn.khuat@gmail.com>
Date: Thu, 29 Sep 2022 14:11:09 +0700
Subject: [PATCH] DateAdd and DateSubtract expression (#25414)
---
.../src/metabase/lib/expressions/config.js | 12 ++
.../lib/expressions/helper-text-strings.ts | 41 ++++++
.../custom-column/cc-data-type.cy.spec.js | 5 +
.../bigquery_cloud_sdk/query_processor.clj | 2 +-
.../mongo/src/metabase/driver/mongo.clj | 18 ++-
.../metabase/driver/mongo/query_processor.clj | 11 ++
.../driver/mongo/query_processor_test.clj | 51 ++++++++
.../vertica/src/metabase/driver/vertica.clj | 12 +-
shared/src/metabase/mbql/normalize.cljc | 8 ++
shared/src/metabase/mbql/schema.cljc | 28 ++++-
shared/test/metabase/mbql/normalize_test.cljc | 20 +--
src/metabase/driver.clj | 7 +-
src/metabase/driver/sql/query_processor.clj | 8 ++
src/metabase/util/date_2.clj | 4 +-
.../middleware/annotate_test.clj | 36 +++---
.../date_time_zone_functions_test.clj | 118 +++++++++++++++++-
16 files changed, 339 insertions(+), 42 deletions(-)
diff --git a/frontend/src/metabase/lib/expressions/config.js b/frontend/src/metabase/lib/expressions/config.js
index a952ecac3af..66f40ae8221 100644
--- a/frontend/src/metabase/lib/expressions/config.js
+++ b/frontend/src/metabase/lib/expressions/config.js
@@ -361,6 +361,16 @@ export const MBQL_CLAUSES = {
type: "number",
args: ["expression"],
},
+ "date-add": {
+ displayName: `dateAdd`,
+ type: "number",
+ args: ["expression", "number", "string"],
+ },
+ "date-subtract": {
+ displayName: `dateSubtract`,
+ type: "number",
+ args: ["expression", "number", "string"],
+ },
};
for (const [name, clause] of Object.entries(MBQL_CLAUSES)) {
@@ -441,6 +451,8 @@ export const EXPRESSION_FUNCTIONS = new Set([
"get-hour",
"get-minute",
"get-second",
+ "date-add",
+ "date-subtract",
// boolean
"contains",
"ends-with",
diff --git a/frontend/src/metabase/lib/expressions/helper-text-strings.ts b/frontend/src/metabase/lib/expressions/helper-text-strings.ts
index 3580e154594..a45981820c6 100644
--- a/frontend/src/metabase/lib/expressions/helper-text-strings.ts
+++ b/frontend/src/metabase/lib/expressions/helper-text-strings.ts
@@ -667,6 +667,47 @@ const helperTextStrings: HelpText[] = [
},
],
},
+ {
+ name: "date-add",
+ structure: "dateAdd(" + t`column` + ", " + t`amount` + ", " + t`unit` + ")",
+ description: t`Adds some units of time to a date or timestamp value.`,
+ example: "dateAdd([" + t`Created At` + '], 1, "' + t`month` + '")',
+ args: [
+ {
+ name: t`column`,
+ description: t`The column with your date or timestamp values.`,
+ },
+ {
+ name: t`amount`,
+ description: t`The number of units to be added.`,
+ },
+ {
+ name: t`unit`,
+ description: t`"year", "month", "quarter", "day", "hour", "minute", "second" or "millisecond".`,
+ },
+ ],
+ },
+ {
+ name: "date-subtract",
+ structure:
+ "dateSubtract(" + t`column` + ", " + t`amount` + ", " + t`unit` + ")",
+ description: t`Subtracts some units of time to a date or timestamp value.`,
+ example: "dateSubtract([" + t`Created At` + '], 1, "' + t`month` + '")',
+ args: [
+ {
+ name: t`column`,
+ description: t`The column with your date or timestamp values.`,
+ },
+ {
+ name: t`amount`,
+ description: t`The number of units to be subtracted.`,
+ },
+ {
+ name: t`unit`,
+ description: t`"year", "month", "quarter", "day", "hour", "minute", "second" or "millisecond".`,
+ },
+ ],
+ },
];
export const getHelpText = (name: string): HelpText | undefined => {
diff --git a/frontend/test/metabase/scenarios/custom-column/cc-data-type.cy.spec.js b/frontend/test/metabase/scenarios/custom-column/cc-data-type.cy.spec.js
index f3e70629074..9b92de27421 100644
--- a/frontend/test/metabase/scenarios/custom-column/cc-data-type.cy.spec.js
+++ b/frontend/test/metabase/scenarios/custom-column/cc-data-type.cy.spec.js
@@ -49,6 +49,11 @@ describe("scenarios > question > custom column > data type", () => {
{ name: "Hour", formula: "hour([Created At])" },
{ name: "Minute", formula: "minute([Created At])" },
{ name: "Second", formula: "second([Created At])" },
+ { name: "Date Add", formula: 'dateAdd([Created At], 1, "month")' },
+ {
+ name: "Date Subtract",
+ formula: 'dateSubtract([Created At], 1, "month")',
+ },
]);
visualize();
diff --git a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk/query_processor.clj b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk/query_processor.clj
index 3461503c7a4..f7665059669 100644
--- a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk/query_processor.clj
+++ b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk/query_processor.clj
@@ -727,7 +727,7 @@
(defmethod sql.qp/cast-temporal-string [:bigquery-cloud-sdk :Coercion/ISO8601->DateTime]
[_driver _semantic_type expr]
- (hx/->timestamp expr))
+ (hx/->datetime expr))
(defmethod sql.qp/cast-temporal-string [:bigquery-cloud-sdk :Coercion/ISO8601->Date]
[_driver _semantic_type expr]
diff --git a/modules/drivers/mongo/src/metabase/driver/mongo.clj b/modules/drivers/mongo/src/metabase/driver/mongo.clj
index 84c511e066e..3f113b0fe9c 100644
--- a/modules/drivers/mongo/src/metabase/driver/mongo.clj
+++ b/modules/drivers/mongo/src/metabase/driver/mongo.clj
@@ -221,12 +221,20 @@
:native-parameters]]
(defmethod driver/supports? [:mongo feature] [_ _] true))
+(defn- db-major-version
+ [db]
+ (some-> (get-in db [:details :version])
+ (str/split #"\.")
+ first
+ Integer/parseInt))
+
(defmethod driver/database-supports? [:mongo :expressions] [_ _ db]
- (let [version (some-> (get-in db [:details :version])
- (str/split #"\.")
- first
- Integer/parseInt)]
- (and (some? version) (<= 4 version))))
+ (let [version (db-major-version db)]
+ (and (some? version) (>= version 4))))
+
+(defmethod driver/database-supports? [:mongo :date-arithmetics] [_ _ db]
+ (let [version (db-major-version db)]
+ (and (some? version) (>= version 5))))
(defmethod driver/mbql->native :mongo
[_ query]
diff --git a/modules/drivers/mongo/src/metabase/driver/mongo/query_processor.clj b/modules/drivers/mongo/src/metabase/driver/mongo/query_processor.clj
index ef8a4aba035..e5a86863fc8 100644
--- a/modules/drivers/mongo/src/metabase/driver/mongo/query_processor.clj
+++ b/modules/drivers/mongo/src/metabase/driver/mongo/query_processor.clj
@@ -455,6 +455,17 @@
(defmethod ->rvalue :coalesce [[_ & args]] {"$ifNull" (mapv ->rvalue args)})
+(defmethod ->rvalue :date-add [[_ inp amount unit]] (do
+ (check-date-operations-supported)
+ {"$dateAdd" {:startDate (->rvalue inp)
+ :unit unit
+ :amount amount}}))
+(defmethod ->rvalue :date-subtract [[_ inp amount unit]] (do
+ (check-date-operations-supported)
+ {"$dateSubtract" {:startDate (->rvalue inp)
+ :unit unit
+ :amount amount}}))
+
;;; +----------------------------------------------------------------------------------------------------------------+
;;; | CLAUSE APPLICATION |
;;; +----------------------------------------------------------------------------------------------------------------+
diff --git a/modules/drivers/mongo/test/metabase/driver/mongo/query_processor_test.clj b/modules/drivers/mongo/test/metabase/driver/mongo/query_processor_test.clj
index 4e1d72e05f4..a81ad8cb25c 100644
--- a/modules/drivers/mongo/test/metabase/driver/mongo/query_processor_test.clj
+++ b/modules/drivers/mongo/test/metabase/driver/mongo/query_processor_test.clj
@@ -2,9 +2,11 @@
(:require [clojure.set :as set]
[clojure.test :refer :all]
[java-time :as t]
+ [metabase.driver :as driver]
[metabase.driver.mongo.query-processor :as mongo.qp]
[metabase.models :refer [Field Table]]
[metabase.query-processor :as qp]
+ [metabase.query-processor-test.date-time-zone-functions-test :as qp.datetime-test]
[metabase.query-processor.timezone :as qp.timezone]
[metabase.test :as mt]
[metabase.util :as u]
@@ -346,3 +348,52 @@
[:interval 1 :year]
[:field "date-field"]]
[:absolute-datetime (t/local-date "2008-05-31")]]))))))
+
+(deftest date-math-tests
+ (mt/test-driver :mongo
+ (mt/dataset qp.datetime-test/times-mixed
+ ;; date arithmetic doesn't supports until mongo 5+
+ (when (driver/database-supports? :mongo :date-arithmetics (mt/db))
+ (testing "date arithmetic with datetime columns"
+ (let [[col-type field-id] [:datetime (mt/id :times :dt)]]
+ (doseq [op [:date-add :date-subtract]
+ unit [:year :quarter :month :day :hour :minute :second :millisecond]
+ {:keys [expected query]}
+ [{:expected [(qp.datetime-test/date-math op #t "2004-03-19 09:19:09" 2 unit col-type)
+ (qp.datetime-test/date-math op #t "2008-06-20 10:20:10" 2 unit col-type)
+ (qp.datetime-test/date-math op #t "2012-11-21 11:21:11" 2 unit col-type)
+ (qp.datetime-test/date-math op #t "2012-11-21 11:21:11" 2 unit col-type)]
+ :query {:expressions {"expr" [op [:field field-id nil] 2 unit]}
+ :fields [[:expression "expr"]]}}
+ {:expected (into [] (frequencies
+ [(qp.datetime-test/date-math op #t "2004-03-19 09:19:09" 2 unit col-type)
+ (qp.datetime-test/date-math op #t "2008-06-20 10:20:10" 2 unit col-type)
+ (qp.datetime-test/date-math op #t "2012-11-21 11:21:11" 2 unit col-type)
+ (qp.datetime-test/date-math op #t "2012-11-21 11:21:11" 2 unit col-type)]))
+ :query {:expressions {"expr" [op [:field field-id nil] 2 unit]}
+ :aggregation [[:count]]
+ :breakout [[:expression "expr"]]}}]]
+ (testing (format "%s %s function works as expected on %s column for driver %s" op unit col-type driver/*driver*)
+ (is (= (set expected) (set (qp.datetime-test/test-date-math query))))))))
+
+ (testing "date arithmetic with date columns"
+ (let [[col-type field-id] [:date (mt/id :times :d)]]
+ (doseq [op [:date-add :date-subtract]
+ unit [:year :quarter :month :day]
+ {:keys [expected query]}
+ [{:expected [(qp.datetime-test/date-math op #t "2004-03-19 00:00:00" 2 unit col-type)
+ (qp.datetime-test/date-math op #t "2008-06-20 00:00:00" 2 unit col-type)
+ (qp.datetime-test/date-math op #t "2012-11-21 00:00:00" 2 unit col-type)
+ (qp.datetime-test/date-math op #t "2012-11-21 00:00:00" 2 unit col-type)]
+ :query {:expressions {"expr" [op [:field field-id nil] 2 unit]}
+ :fields [[:expression "expr"]]}}
+ {:expected (into [] (frequencies
+ [(qp.datetime-test/date-math op #t "2004-03-19 00:00:00" 2 unit col-type)
+ (qp.datetime-test/date-math op #t "2008-06-20 00:00:00" 2 unit col-type)
+ (qp.datetime-test/date-math op #t "2012-11-21 00:00:00" 2 unit col-type)
+ (qp.datetime-test/date-math op #t "2012-11-21 00:00:00" 2 unit col-type)]))
+ :query {:expressions {"expr" [op [:field field-id nil] 2 unit]}
+ :aggregation [[:count]]
+ :breakout [[:expression "expr"]]}}]]
+ (testing (format "%s %s function works as expected on %s column for driver %s" op unit col-type driver/*driver*)
+ (is (= (set expected) (set (qp.datetime-test/test-date-math query))))))))))))
diff --git a/modules/drivers/vertica/src/metabase/driver/vertica.clj b/modules/drivers/vertica/src/metabase/driver/vertica.clj
index 7488b3d2170..c756a52be5a 100644
--- a/modules/drivers/vertica/src/metabase/driver/vertica.clj
+++ b/modules/drivers/vertica/src/metabase/driver/vertica.clj
@@ -123,11 +123,17 @@
(defmethod sql.qp/add-interval-honeysql-form :vertica
[_ hsql-form amount unit]
+ (hsql/call :timestampadd unit)
+ ;; using `timestampadd` instead of `+ (INTERVAL)` because vertica add inteval for month, or year
+ ;; by adding the equivalent number of days, not adding the unit compoinent.
+ ;; For example `select date '2004-02-02' + interval '1 year' will return `2005-02-01` because it's adding
+ ;; 365 days under the hood and 2004 is a leap year. Whereas other dbs will return `2006-02-02`.
+ ;; So we use timestampadd to make the behavior consistent with other dbs
(let [acceptable-types (case unit
- (:millisecond :second :minute :hour) #{"time" "timetz" "timestamp" "timestamptz"}
- (:day :week :month :quarter :year) #{"date" "timestamp" "timestamptz"})
+ (:millisecond :second :minute :hour) #{"time" "timetz" "timestamp" "timestamptz"}
+ (:day :week :month :quarter :year) #{"date" "timestamp" "timestamptz"})
hsql-form (hx/cast-unless-type-in "timestamp" acceptable-types hsql-form)]
- (hx/+ hsql-form (hsql/raw (format "(INTERVAL '%d %s')" (int amount) (name unit))))))
+ (hsql/call :timestampadd unit amount hsql-form)))
(defn- materialized-views
"Fetch the Materialized Views for a Vertica `database`.
diff --git a/shared/src/metabase/mbql/normalize.cljc b/shared/src/metabase/mbql/normalize.cljc
index ebb400e2c24..0fce777080f 100644
--- a/shared/src/metabase/mbql/normalize.cljc
+++ b/shared/src/metabase/mbql/normalize.cljc
@@ -149,6 +149,14 @@
[[_ amount unit]]
[:interval amount (maybe-normalize-token unit)])
+(defmethod normalize-mbql-clause-tokens :date-add
+ [[_ field amount unit]]
+ [:date-add (normalize-tokens field :ignore-path) amount (maybe-normalize-token unit)])
+
+(defmethod normalize-mbql-clause-tokens :date-subtract
+ [[_ field amount unit]]
+ [:date-subtract (normalize-tokens field :ignore-path) amount (maybe-normalize-token unit)])
+
(defmethod normalize-mbql-clause-tokens :temporal-extract
[[_ field unit]]
[:temporal-extract (normalize-tokens field :ignore-path) (maybe-normalize-token unit)])
diff --git a/shared/src/metabase/mbql/schema.cljc b/shared/src/metabase/mbql/schema.cljc
index 2ab06654bfc..3bf972118cf 100644
--- a/shared/src/metabase/mbql/schema.cljc
+++ b/shared/src/metabase/mbql/schema.cljc
@@ -475,9 +475,13 @@
;; SUGAR drivers do not need to implement
:get-year :get-quarter :get-month :get-day :get-day-of-week :get-hour :get-minute :get-second})
+(def date-arithmetic-functions
+ "Functions to do math with date, datetime."
+ #{:date-add :date-subtract})
+
(def date+time+timezone-functions
"Date, time, and timezone related functions."
- (set/union temporal-extract-functions))
+ (set/union temporal-extract-functions date-arithmetic-functions))
(declare ArithmeticExpression)
(declare BooleanExpression)
@@ -512,6 +516,10 @@
(partial is-clause? :value)
value
+ ;; Recursively doing date math
+ (partial is-clause? date-arithmetic-functions)
+ (s/recursive #'DatetimeExpression)
+
:else
Field))
@@ -651,8 +659,23 @@
(defclause ^{:requires-features #{:temporal-extract}} ^:sugar get-second
datetime DateTimeExpressionArg)
+(def ^:private ArithmeticDateTimeUnit
+ (s/named
+ (apply s/enum #{:millisecond :second :minute :hour :day :week :month :quarter :year})
+ "arithmetic-datetime-unit"))
+
+(defclause ^{:requires-features #{:date-arithmetics}} date-add
+ datetime DateTimeExpressionArg
+ amount NumericExpressionArg
+ unit ArithmeticDateTimeUnit)
+
+(defclause ^{:requires-features #{:date-arithmetics}} date-subtract
+ datetime DateTimeExpressionArg
+ amount NumericExpressionArg
+ unit ArithmeticDateTimeUnit)
+
(def ^:private DatetimeExpression*
- (one-of temporal-extract
+ (one-of temporal-extract date-add date-subtract
;; SUGAR drivers do not need to implement
get-year get-quarter get-month get-day get-day-of-week get-hour
get-minute get-second))
@@ -915,7 +938,6 @@
;; SUGAR clauses
cum-count count)))
-
(def ^:private UnnamedAggregation
(s/recursive #'UnnamedAggregation*))
diff --git a/shared/test/metabase/mbql/normalize_test.cljc b/shared/test/metabase/mbql/normalize_test.cljc
index 78a49a78cc8..376e3d49087 100644
--- a/shared/test/metabase/mbql/normalize_test.cljc
+++ b/shared/test/metabase/mbql/normalize_test.cljc
@@ -565,7 +565,13 @@
{:query {:expressions {"prev_month" [:+ [:field-id 13] [:interval -1 :month]]}}}
{:query {:expressions {:prev_month ["-" ["field-id" 13] ["interval" 1 "month"] ["interval" 1 "day"]]}}}
- {:query {:expressions {"prev_month" [:- [:field-id 13] [:interval 1 :month] [:interval 1 :day]]}}}}
+ {:query {:expressions {"prev_month" [:- [:field-id 13] [:interval 1 :month] [:interval 1 :day]]}}}
+
+ {:query {:expressions {:date-add ["date-add" ["field" 1 nil] 1 "month"]}}}
+ {:query {:expressions {"date-add" [:date-add [:field 1 nil] 1 :month]}}}
+
+ {:query {:expressions {:date-subtract ["date-subtract" ["field" 1 nil] 1 "month"]}}}
+ {:query {:expressions {"date-subtract" [:date-subtract [:field 1 nil] 1 :month]}}}}
"expressions handle namespaced keywords correctly"
{{:query {"expressions" {:abc/def ["+" 1 2]}
@@ -1179,12 +1185,12 @@
(mbql.normalize/normalize
{:database 1
:native {:template-tags {"name" {:id "1f56330b-3dcb-75a3-8f3d-5c2c2792b749"
- :name "name"
- :display-name "Name"
- :type "dimension"
- :dimension ["field" 14 nil]
- :widget-type "string/="
- :default ["Hudson Borer"]}}
+ :name "name"
+ :display-name "Name"
+ :type "dimension"
+ :dimension ["field" 14 nil]
+ :widget-type "string/="
+ :default ["Hudson Borer"]}}
:query "select * from PEOPLE where {{name}}"}
:type "native"
:parameters []})))))
diff --git a/src/metabase/driver.clj b/src/metabase/driver.clj
index 5f37b241735..d20e028b924 100644
--- a/src/metabase/driver.clj
+++ b/src/metabase/driver.clj
@@ -436,10 +436,14 @@
;; Does the driver support percentile calculations (including median)
:percentile-aggregations
- ;; Does the driver support date, time, and timezone manipulation functions?
+ ;; Does the driver support date extraction functions? (i.e get year component of a datetime column)
;; DEFAULTS TO TRUE
:temporal-extract
+ ;; Does the driver support doing math with datetime? (i.e Adding 1 year to a datetime column)
+ ;; DEFAULTS TO TRUE
+ :date-arithmetics
+
;; Does the driver support experimental "writeback" actions like "delete this row" or "insert a new row" from 44+?
:actions
@@ -466,6 +470,7 @@
(defmethod supports? [::driver :basic-aggregations] [_ _] true)
(defmethod supports? [::driver :case-sensitivity-string-filter-options] [_ _] true)
+(defmethod supports? [::driver :date-arithmetics] [_ _] true)
(defmethod supports? [::driver :temporal-extract] [_ _] true)
(defmulti database-supports?
diff --git a/src/metabase/driver/sql/query_processor.clj b/src/metabase/driver/sql/query_processor.clj
index de69ec71f9c..43d9a3aba87 100644
--- a/src/metabase/driver/sql/query_processor.clj
+++ b/src/metabase/driver/sql/query_processor.clj
@@ -590,6 +590,14 @@
[driver [_ arg unit]]
(date driver (temporal-extract-unit->date-unit unit) (->honeysql driver arg)))
+(defmethod ->honeysql [:sql :date-add]
+ [driver [_ arg amount unit]]
+ (add-interval-honeysql-form driver (->honeysql driver arg) amount unit))
+
+(defmethod ->honeysql [:sql :date-subtract]
+ [driver [_ arg amount unit]]
+ (add-interval-honeysql-form driver (->honeysql driver arg) (- amount) unit))
+
;;; +----------------------------------------------------------------------------------------------------------------+
;;; | Field Aliases (AS Forms) |
;;; +----------------------------------------------------------------------------------------------------------------+
diff --git a/src/metabase/util/date_2.clj b/src/metabase/util/date_2.clj
index 98a4c396122..120a81bbb6f 100644
--- a/src/metabase/util/date_2.clj
+++ b/src/metabase/util/date_2.clj
@@ -186,8 +186,8 @@
:quarter (t/months (* amount 3))
:year (t/years amount))))))
-;; TIMEZONE FIXME - we should add `:millisecond-of-second` (or `:fraction-of-second`?) and `:second-of-minute` as
-;; well. Not sure where we'd use these, but we should have them for consistency
+;; TIMEZONE FIXME - we should add `:millisecond-of-second` (or `:fraction-of-second`?) .
+;; Not sure where we'd use these, but we should have them for consistency
(def extract-units
"Units which return a (numerical, periodic) component of a date"
#{:second-of-minute
diff --git a/test/metabase/query_processor/middleware/annotate_test.clj b/test/metabase/query_processor/middleware/annotate_test.clj
index 9a27c4afe6c..12c2d2c9204 100644
--- a/test/metabase/query_processor/middleware/annotate_test.clj
+++ b/test/metabase/query_processor/middleware/annotate_test.clj
@@ -209,25 +209,25 @@
(testing "For fields with parents we should return them with a combined name including parent's name"
(tt/with-temp* [Field [parent {:name "parent", :table_id (mt/id :venues)}]
Field [child {:name "child", :table_id (mt/id :venues), :parent_id (u/the-id parent)}]]
- (mt/with-everything-store
+ (mt/with-everything-store
(is (= {:description nil
- :table_id (mt/id :venues)
- :semantic_type nil
- :effective_type nil
- ;; these two are a gross symptom. there's some tension. sometimes it makes sense to have an effective
- ;; type: the db type is different and we have a way to convert. Othertimes, it doesn't make sense:
- ;; when the info is inferred. the solution to this might be quite extensive renaming
- :coercion_strategy nil
- :name "parent.child"
- :settings nil
- :field_ref [:field (u/the-id child) nil]
- :nfc_path nil
- :parent_id (u/the-id parent)
- :id (u/the-id child)
- :visibility_type :normal
- :display_name "Child"
- :fingerprint nil
- :base_type :type/Text}
+ :table_id (mt/id :venues)
+ :semantic_type nil
+ :effective_type nil
+ ;; these two are a gross symptom. there's some tension. sometimes it makes sense to have an effective
+ ;; type: the db type is different and we have a way to convert. Othertimes, it doesn't make sense:
+ ;; when the info is inferred. the solution to this might be quite extensive renaming
+ :coercion_strategy nil
+ :name "parent.child"
+ :settings nil
+ :field_ref [:field (u/the-id child) nil]
+ :nfc_path nil
+ :parent_id (u/the-id parent)
+ :id (u/the-id child)
+ :visibility_type :normal
+ :display_name "Child"
+ :fingerprint nil
+ :base_type :type/Text}
(into {} (#'annotate/col-info-for-field-clause {} [:field (u/the-id child) nil])))))))
(testing "nested-nested fields should include grandparent name (etc)"
diff --git a/test/metabase/query_processor_test/date_time_zone_functions_test.clj b/test/metabase/query_processor_test/date_time_zone_functions_test.clj
index 943425e0727..8f1979b2f69 100644
--- a/test/metabase/query_processor_test/date_time_zone_functions_test.clj
+++ b/test/metabase/query_processor_test/date_time_zone_functions_test.clj
@@ -1,9 +1,15 @@
(ns metabase.query-processor-test.date-time-zone-functions-test
- (:require [clojure.test :refer :all]
+ (:require [clojure.string :as str]
+ [clojure.test :refer :all]
+ [java-time :as t]
[metabase.driver :as driver]
[metabase.test :as mt]
[metabase.util.date-2 :as u.date]))
+;;; +----------------------------------------------------------------------------------------------------------------+
+;;; | Date extract tests |
+;;; +----------------------------------------------------------------------------------------------------------------+
+
(defn test-temporal-extract
[{:keys [aggregation breakout expressions fields filter limit]}]
(if breakout
@@ -90,7 +96,6 @@
(testing (format "extract %s function works as expected on %s column for driver %s" op col-type driver/*driver*)
(is (= (set (expected-fn op)) (set (test-temporal-extract (query-fn op field-id)))))))))
- ;; need to have seperate tests for mongo because it doesn't have supports for casting yet
(mt/test-driver :mongo
(testing "with datetimes columns"
(let [[col-type field-id] [:datetime (mt/id :times :dt)]]
@@ -142,3 +147,112 @@
:fields [[:field (mt/id :times :index) nil]]}}]]
(testing title
(is (= expected (test-temporal-extract query))))))))
+
+;;; +----------------------------------------------------------------------------------------------------------------+
+;;; | Date arithmetics tests |
+;;; +----------------------------------------------------------------------------------------------------------------+
+
+(defn date-math
+ [op x amount unit col-type]
+ (let [amount (if (= op :date-add)
+ amount
+ (- amount))
+ fmt (cond
+ ;; the :date column of :presto should have this format too,
+ ;; but the test data we created for presto is datetime even if we define it as date
+ (and (= driver/*driver* :presto) (#{:text-as-date} col-type))
+ "yyyy-MM-dd"
+
+ (= unit :millisecond)
+ "yyyy-MM-dd HH:mm:ss.SSS"
+
+ :else
+ "yyyy-MM-dd HH:mm:ss")]
+ (t/format fmt (u.date/add x unit amount))))
+
+(defn- normalize-timestamp-str [x]
+ (if (number? x)
+ (int x)
+ (-> x
+ (str/replace #"T" " ")
+ (str/replace #"Z" ""))))
+
+(defn test-date-math
+ [{:keys [aggregation breakout expressions fields filter limit]}]
+ (if breakout
+ (->> (mt/run-mbql-query times {:expressions expressions
+ :aggregation aggregation
+ :limit limit
+ :filter filter
+ :breakout breakout})
+ (mt/formatted-rows [normalize-timestamp-str normalize-timestamp-str]))
+ (->> (mt/run-mbql-query times {:expressions expressions
+ :aggregation aggregation
+ :limit limit
+ :filter filter
+ :fields fields})
+ (mt/formatted-rows [normalize-timestamp-str])
+ (map first))))
+
+(deftest date-math-tests
+ (mt/dataset times-mixed
+ ;; mongo doesn't supports coercion yet so we exclude it here, Tests for it are in [[metabase.driver.mongo.query-processor-test]]
+ (mt/test-drivers (disj (mt/normal-drivers-with-feature :date-arithmetics) :mongo)
+ (testing "date arithmetic with datetime columns"
+ (doseq [[col-type field-id] [[:datetime (mt/id :times :dt)] [:text-as-datetime (mt/id :times :as_dt)]]
+ op [:date-add :date-subtract]
+ unit [:year :quarter :month :day :hour :minute :second]
+
+ {:keys [expected query]}
+ [{:expected [(date-math op #t "2004-03-19 09:19:09" 2 unit col-type) (date-math op #t "2008-06-20 10:20:10" 2 unit col-type)
+ (date-math op #t "2012-11-21 11:21:11" 2 unit col-type) (date-math op #t "2012-11-21 11:21:11" 2 unit col-type)]
+ :query {:expressions {"expr" [op [:field field-id nil] 2 unit]}
+ :fields [[:expression "expr"]]}}
+ {:expected (into [] (frequencies
+ [(date-math op #t "2004-03-19 09:19:09" 2 unit col-type) (date-math op #t "2008-06-20 10:20:10" 2 unit col-type)
+ (date-math op #t "2012-11-21 11:21:11" 2 unit col-type) (date-math op #t "2012-11-21 11:21:11" 2 unit col-type)]))
+ :query {:expressions {"expr" [op [:field field-id nil] 2 unit]}
+ :aggregation [[:count]]
+ :breakout [[:expression "expr"]]}}]]
+ (testing (format "%s %s function works as expected on %s column for driver %s" op unit col-type driver/*driver*)
+ (is (= (set expected) (set (test-date-math query)))))))
+
+ (testing "date arithmetic with datetime columns"
+ (doseq [[col-type field-id] [[:date (mt/id :times :d)] [:text-as-date (mt/id :times :as_d)]]
+ op [:date-add :date-subtract]
+ unit [:year :quarter :month :day]
+
+ {:keys [expected query]}
+ [{:expected [(date-math op #t "2004-03-19 00:00:00" 2 unit col-type) (date-math op #t "2008-06-20 00:00:00" 2 unit col-type)
+ (date-math op #t "2012-11-21 00:00:00" 2 unit col-type) (date-math op #t "2012-11-21 00:00:00" 2 unit col-type)]
+ :query {:expressions {"expr" [op [:field field-id nil] 2 unit]}
+ :fields [[:expression "expr"]]}}
+ {:expected (into [] (frequencies
+ [(date-math op #t "2004-03-19 00:00:00" 2 unit col-type) (date-math op #t "2008-06-20 00:00:00" 2 unit col-type)
+ (date-math op #t "2012-11-21 00:00:00" 2 unit col-type) (date-math op #t "2012-11-21 00:00:00" 2 unit col-type)]))
+ :query {:expressions {"expr" [op [:field field-id nil] 2 unit]}
+ :aggregation [[:count]]
+ :breakout [[:expression "expr"]]}}]]
+ (testing (format "%s %s function works as expected on %s column for driver %s" op unit col-type driver/*driver*)
+ (is (= (set expected) (set (test-date-math query))))))))))
+
+(deftest date-math-with-extract-test
+ (mt/test-drivers (mt/normal-drivers-with-feature :date-arithmetics)
+ (mt/dataset times-mixed
+ (doseq [{:keys [title expected query]}
+ [{:title "Nested date math then extract"
+ :expected [2006 2010 2014]
+ :query {:expressions {"expr" [:get-year [:date-add [:field (mt/id :times :dt) nil] 2 :year]]}
+ :fields [[:expression "expr"]]}}
+
+ {:title "Nested date math twice"
+ :expected ["2006-05-19 09:19:09" "2010-08-20 10:20:10" "2015-01-21 11:21:11"]
+ :query {:expressions {"expr" [:date-add [:date-add [:field (mt/id :times :dt) nil] 2 :year] 2 :month]}
+ :fields [[:expression "expr"]]}}
+
+ {:title "filter with date math"
+ :expected [1]
+ :query {:filter [:= [:get-year [:date-add [:field (mt/id :times :dt) nil] 2 :year]] 2006]
+ :fields [[:field (mt/id :times :index)]]}}]]
+ (testing title
+ (is (= (set expected) (set (test-date-math query)))))))))
--
GitLab