Skip to content
Snippets Groups Projects
Unverified Commit 0becaab7 authored by Cam Saul's avatar Cam Saul Committed by GitHub
Browse files

Use java.time classes everywhere (#11390)

parent c0f9d8de
No related branches found
No related tags found
No related merge requests found
Showing
with 194 additions and 18177 deletions
......@@ -21,7 +21,6 @@
(let-404 1)
(match 1)
(merge-with 1)
(with-redefs-fn 1)
(p.types/defprotocol+ '(1 (:defn)))
(p.types/def-abstract-type '(1 (:defn)))
(p.types/deftype+ '(2 nil nil (:defn)))
......
......@@ -106,7 +106,6 @@
call-with-paused-query
discard-setting-changes
doall-recursive
exception-and-message
is-uuid-string?
metabase-logger
postwalk-pred
......
......@@ -434,8 +434,8 @@ Object {
},
"type": Object {
"type/DateTime": Object {
"earliest": "2016-05-01T01:56:13.352Z",
"latest": "2020-04-19T21:07:15.657Z",
"earliest": "2016-04-30T18:56:13.352",
"latest": "2020-04-19T14:07:15.657",
},
},
},
......@@ -1573,8 +1573,8 @@ Object {
},
"type": Object {
"type/DateTime": Object {
"earliest": "1958-04-26T08:00:00.000Z",
"latest": "2000-04-03T07:00:00.000Z",
"earliest": "1958-04-26",
"latest": "2000-04-03",
},
},
},
......@@ -1793,8 +1793,8 @@ Object {
},
"type": Object {
"type/DateTime": Object {
"earliest": "2016-04-20T04:35:18.752Z",
"latest": "2019-04-19T21:06:27.300Z",
"earliest": "2016-04-19T21:35:18.752",
"latest": "2019-04-19T14:06:27.3",
},
},
},
......@@ -2778,8 +2778,8 @@ Object {
},
"type": Object {
"type/DateTime": Object {
"earliest": "2016-04-27T02:29:55.147Z",
"latest": "2019-04-15T20:34:19.931Z",
"earliest": "2016-04-26T19:29:55.147",
"latest": "2019-04-15T13:34:19.931",
},
},
},
......@@ -3443,7 +3443,7 @@ Object {
},
"type": Object {
"type/Text": Object {
"average-length": 177.41996402877697,
"average-length": 182.37589928057554,
"percent-email": 0,
"percent-json": 0,
"percent-url": 0,
......@@ -3626,8 +3626,8 @@ Object {
},
"type": Object {
"type/DateTime": Object {
"earliest": "2016-06-03T07:37:05.818Z",
"latest": "2020-04-19T21:15:25.677Z",
"earliest": "2016-06-03T00:37:05.818",
"latest": "2020-04-19T14:15:25.677",
},
},
},
......
This diff is collapsed.
This diff is collapsed.
(ns metabase.driver.bigquery.query-processor-test
(:require [clj-time.core :as time]
[clojure.test :refer :all]
(:require [clojure.test :refer :all]
[honeysql.core :as hsql]
[java-time :as t]
[metabase
......@@ -150,7 +149,7 @@
"A UTC date is returned, we should read/return it as UTC")
(is (= "2018-08-31T00:00:00-05:00"
(tu.tz/with-jvm-tz (time/time-zone-for-id "America/Chicago")
(tu.tz/with-system-timezone-id "America/Chicago"
(tt/with-temp* [Database [db {:engine :bigquery
:details (assoc (:details (data/db))
:use-jvm-timezone true)}]]
......@@ -160,7 +159,7 @@
"the correct date is compared"))
(is (= "2018-08-31T00:00:00+07:00"
(tu.tz/with-jvm-tz (time/time-zone-for-id "Asia/Jakarta")
(tu.tz/with-system-timezone-id "Asia/Jakarta"
(tt/with-temp* [Database [db {:engine :bigquery
:details (assoc (:details (data/db))
:use-jvm-timezone true)}]]
......
......@@ -14,7 +14,6 @@
[interface :as tx]
[sql :as sql.tx]]
[metabase.util
[date :as du]
[date-2 :as u.date]
[schema :as su]]
[schema.core :as s])
......@@ -130,7 +129,7 @@
"Convert the HoneySQL form we normally use to wrap a `Timestamp` to a Google `DateTime`."
[{[{s :literal}] :args}]
{:pre [(string? s) (seq s)]}
(DateTime. (du/->Timestamp (str/replace s #"'" ""))))
(DateTime. (t/to-java-date (u.date/parse (str/replace s #"'" "")))))
(defn- insert-data! [^String dataset-id, ^String table-id, row-maps]
......
(ns metabase.driver.druid-test
(:require [cheshire.core :as json]
[clj-time.core :as time]
[clojure.test :refer :all]
[expectations :refer [expect]]
[java-time :as t]
......@@ -51,7 +50,7 @@
(tu/with-temporary-setting-values [report-timezone "America/Los_Angeles"]
(is (= expected
(table-rows-sample))))
(tu.tz/with-jvm-tz (time/time-zone-for-id "America/Chicago")
(tu.tz/with-system-timezone-id "America/Chicago"
(is (= expected
(table-rows-sample))))))))
......
......@@ -7,12 +7,16 @@
[metabase.mbql.util :as mbql.u]
[metabase.query-processor.store :as qp.store]
[metabase.util
[date :as du]
[date-2 :as u.date]
[i18n :as ui18n :refer [deferred-tru tru]]
[schema :as su]]
[metabase.util.date-2.parse :as u.date.parse]
[metabase.util.date-2.parse.builder :as u.date.parse.builder]
[schema.core :as s])
(:import [com.google.api.services.analytics.model GaData GaData$ColumnHeaders]))
(:import [com.google.api.services.analytics.model GaData GaData$ColumnHeaders]
java.time.DayOfWeek
java.time.format.DateTimeFormatter
org.threeten.extra.YearWeek))
(def ^:private ^:const earliest-date "2005-01-01")
(def ^:private ^:const latest-date "today")
......@@ -407,16 +411,26 @@
(defn- parse-number [s]
(edn/read-string (str/replace s #"^0+(.+)$" "$1")))
(def ^:private ^DateTimeFormatter iso-year-week-formatter
(u.date.parse.builder/formatter
(u.date.parse.builder/value :iso/week-based-year 4)
(u.date.parse.builder/value :iso/week-of-week-based-year 2)))
(defn- parse-iso-year-week [^String s]
(when s
(-> (YearWeek/from (.parse iso-year-week-formatter s))
(.atDay DayOfWeek/MONDAY))))
(def ^:private ga-dimension->date-format-fn
{"ga:minute" parse-number
"ga:dateHour" (partial du/parse-date "yyyyMMddHH")
"ga:dateHour" (partial u.date.parse/parse-with-formatter "yyyyMMddHH")
"ga:hour" parse-number
"ga:date" (partial du/parse-date "yyyyMMdd")
"ga:date" (partial u.date.parse/parse-with-formatter "yyyyMMdd")
"ga:dayOfWeek" (comp inc parse-number)
"ga:day" parse-number
"ga:isoYearIsoWeek" (partial du/parse-date "xxxxww")
"ga:isoYearIsoWeek" parse-iso-year-week
"ga:week" parse-number
"ga:yearMonth" (partial du/parse-date "yyyyMM")
"ga:yearMonth" (partial u.date.parse/parse-with-formatter "yyyyMM")
"ga:month" parse-number
"ga:year" parse-number})
......
......@@ -18,7 +18,7 @@
(deftest iso-year-iso-week-test
(testing "Make sure we properly parse isoYearIsoWeeks (#9244)"
(is (= #inst "2018-12-31T00:00:00.000000000-00:00"
(is (= #t "2018-12-31"
((#'ga.qp/ga-dimension->date-format-fn "ga:isoYearIsoWeek") "201901")))))
(deftest filter-test
......
......@@ -4,6 +4,7 @@
[core :as json]
[generate :as json.generate]]
[clojure.tools.logging :as log]
[java-time :as t]
[metabase
[driver :as driver]
[util :as u]]
......@@ -12,18 +13,26 @@
[metabase.driver.mongo
[query-processor :as qp]
[util :refer [with-mongo-connection]]]
[metabase.query-processor.store :as qp.store]
[metabase.plugins.classloader :as classloader]
[metabase.query-processor
[store :as qp.store]
[timezone :as qp.timezone]]
[monger
[collection :as mc]
[command :as cmd]
[conversion :as conv]
[conversion :as m.conversion]
[db :as mdb]]
[schema.core :as s]
[taoensso.nippy :as nippy])
(:import com.mongodb.DB
[java.time Instant LocalDate LocalDateTime LocalTime OffsetDateTime OffsetTime ZonedDateTime]
org.bson.BsonUndefined
org.bson.types.ObjectId))
;; See http://clojuremongodb.info/articles/integration.html Loading this namespace will load appropriate Monger
;; integrations with Cheshire.
(classloader/require 'monger.json)
;; JSON Encoding (etc.)
;; Encode BSON undefined like `nil`
......@@ -43,7 +52,7 @@
[_ details]
(with-mongo-connection [^DB conn, details]
(= (float (-> (cmd/db-stats conn)
(conv/from-db-object :keywordize)
(m.conversion/from-db-object :keywordize)
:ok))
1.0)))
......@@ -199,3 +208,43 @@
(defmethod driver/execute-query :mongo
[_ query]
(qp/execute-query query))
;; It seems to be the case that the only thing BSON supports is DateTime which is basically the equivalent of Instant;
;; for the rest of the types, we'll have to fake it
(extend-protocol m.conversion/ConvertToDBObject
Instant
(to-db-object [t]
(org.bson.BsonDateTime. (t/to-millis-from-epoch t)))
LocalDate
(to-db-object [t]
(m.conversion/to-db-object (t/local-date-time t (t/local-time 0))))
LocalDateTime
(to-db-object [t]
;; QP store won't be bound when loading test data for example.
(m.conversion/to-db-object (t/instant t (t/zone-id (try
(qp.timezone/results-timezone-id)
(catch Throwable _
"UTC"))))))
LocalTime
(to-db-object [t]
(m.conversion/to-db-object (t/local-date-time (t/local-date "1970-01-01") t)))
OffsetDateTime
(to-db-object [t]
(m.conversion/to-db-object (t/instant t)))
OffsetTime
(to-db-object [t]
(m.conversion/to-db-object (t/offset-date-time (t/local-date "1970-01-01") t (t/zone-offset t))))
ZonedDateTime
(to-db-object [t]
(m.conversion/to-db-object (t/instant t))))
(extend-protocol m.conversion/ConvertFromDBObject
java.util.Date
(from-db-object [t _]
(t/instant t)))
......@@ -25,18 +25,11 @@
[schema :as su]]
[monger
[collection :as mc]
[conversion :as m.conversion]
json
[operators :refer :all]]
[schema.core :as s])
(:import metabase.models.field.FieldInstance
org.bson.types.ObjectId))
;; See http://clojuremongodb.info/articles/integration.html Loading this namespace will load appropriate Monger
;; integrations with Cheshire. The comment below is to fool `cljr-clean-ns` into keeping the namespace in the
;; `:require` form above
(comment monger.json/keep-me)
;;; +----------------------------------------------------------------------------------------------------------------+
;;; | Schema |
;;; +----------------------------------------------------------------------------------------------------------------+
......@@ -75,21 +68,6 @@
column names in the query (?)"
[s/Keyword])
;; TIMEZONE FIXME — or use codecs — https://mongodb.github.io/mongo-java-driver/3.0/bson/codecs/
#_(extend-protocol m.conversion/ConvertToDBObject
java.time.LocalDate
(to-db-object [t])
java.time.LocalTime
java.time.LocalDateTime
java.time.OffsetTime
java.time.OffsetDateTime
java.time.ZonedDateTime)
(extend-protocol m.conversion/ConvertFromDBObject
java.util.Date
(from-db-object [t _]
(t/instant t)))
;;; +----------------------------------------------------------------------------------------------------------------+
;;; | QP Impl |
;;; +----------------------------------------------------------------------------------------------------------------+
......
......@@ -26,20 +26,12 @@
(keyword (:field-name field-definition)))]
;; Use map-indexed so we can get an ID for each row (index + 1)
(doseq [[i row] (map-indexed (partial vector) rows)]
(let [row (for [v row]
;; Conver all the java.sql.Timestamps to java.util.Date, because the Mongo driver insists on
;; being obnoxious and going from using Timestamps in 2.x to Dates in 3.x
;;
;; TIMEZONE FIXME — stop using legacy types
(if (instance? java.sql.Timestamp v)
(java.util.Date. (.getTime ^java.sql.Timestamp v))
v))]
(try
;; Insert each row
(mc/insert mongo-db (name table-name) (assoc (zipmap field-names row)
:_id (inc i)))
;; If row already exists then nothing to do
(catch com.mongodb.MongoException _))))))))
(try
;; Insert each row
(mc/insert mongo-db (name table-name) (assoc (zipmap field-names row)
:_id (inc i)))
;; If row already exists then nothing to do
(catch com.mongodb.MongoException _)))))))
(defmethod tx/format-name :mongo
[_ table-or-field-name]
......
......@@ -74,12 +74,7 @@
(when sid
(str ":" sid))
(when service-name
(str "/" service-name)))
;; By default the Oracle JDBC driver isn't compliant with JDBC standards -- instead of returning types like
;; java.sql.Timestamp it returns wacky types like oracle.sql.TIMESTAMPT. By setting this property the JDBC driver
;; will return the appropriate types. See this page for more details:
;; http://docs.oracle.com/database/121/JJDBC/datacc.htm#sthref437
:oracle.jdbc.J2EE13Compliant true}
(str "/" service-name)))}
(dissoc details :host :port :sid :service-name)))
(defmethod driver/can-connect? :oracle
......
......@@ -28,36 +28,32 @@
[toucan.util.test :as tt]))
(deftest connection-details->spec-test
(are [message expected-spec details] (is (= expected-spec
(sql-jdbc.conn/connection-details->spec :oracle details))
message)
"You should be able to connect with an SID"
{:classname "oracle.jdbc.OracleDriver"
:subprotocol "oracle:thin"
:subname "@localhost:1521:ORCL"
:oracle.jdbc.J2EE13Compliant true}
{:host "localhost"
:port 1521
:sid "ORCL"}
"You should be able to specify a Service Name with no SID"
{:classname "oracle.jdbc.OracleDriver"
:subprotocol "oracle:thin"
:subname "@localhost:1521/MyCoolService"
:oracle.jdbc.J2EE13Compliant true}
{:host "localhost"
:port 1521
:service-name "MyCoolService"}
"You should be able to specifiy a Service Name *and* an SID"
{:classname "oracle.jdbc.OracleDriver"
:subprotocol "oracle:thin"
:subname "@localhost:1521:ORCL/MyCoolService"
:oracle.jdbc.J2EE13Compliant true}
{:host "localhost"
:port 1521
:service-name "MyCoolService"
:sid "ORCL"}))
(doseq [[message expected-spec details]
[["You should be able to connect with an SID"
{:classname "oracle.jdbc.OracleDriver"
:subprotocol "oracle:thin"
:subname "@localhost:1521:ORCL"}
{:host "localhost"
:port 1521
:sid "ORCL"}]
["You should be able to specify a Service Name with no SID"
{:classname "oracle.jdbc.OracleDriver"
:subprotocol "oracle:thin"
:subname "@localhost:1521/MyCoolService"}
{:host "localhost"
:port 1521
:service-name "MyCoolService"}]
["You should be able to specifiy a Service Name *and* an SID"
{:classname "oracle.jdbc.OracleDriver"
:subprotocol "oracle:thin"
:subname "@localhost:1521:ORCL/MyCoolService"}
{:host "localhost"
:port 1521
:service-name "MyCoolService"
:sid "ORCL"}]]]
(is (= expected-spec
(sql-jdbc.conn/connection-details->spec :oracle details))
message)))
;; no SID and not Service Name should throw an exception
(expect
......
......@@ -13,7 +13,8 @@
[datasets :as datasets :refer [expect-with-driver]]
[interface :as tx]
[sql :as sql.tx]]
[metabase.test.data.sql.ddl :as ddl]))
[metabase.test.data.sql.ddl :as ddl]
[metabase.test.util.log :as tu.log]))
;; make sure we didn't break the code that is used to generate DDL statements when we add new test datasets
(deftest ddl-statements-test
......@@ -90,11 +91,12 @@
(deftest can-connect-test
(datasets/test-driver :snowflake
(let [can-connect? (fn [details]
(driver/can-connect? :snowflake details))]
(letfn [(can-connect? [details]
(driver/can-connect? :snowflake details))]
(is (= true
(can-connect? (:details (data/db))))
"can-connect? should return true for normal Snowflake DB details")
(is (= false
(can-connect? (assoc (:details (data/db)) :db (tu/random-name))))
(tu.log/suppress-output
(can-connect? (assoc (:details (data/db)) :db (tu/random-name)))))
"can-connect? should return false for Snowflake databases that don't exist (#9041)"))))
......@@ -111,5 +111,6 @@
(defmethod tx/id-field-type :snowflake [_] :type/Number)
(defmethod load-data/load-data! :snowflake [& args]
(defmethod load-data/load-data! :snowflake
[& args]
(apply load-data/load-data-add-ids! args))
......@@ -130,9 +130,9 @@
"Run the query itself."
[driver {sql :query, :keys [params remark max-rows]} connection]
(let [sql (str "-- " remark "\n" sql)
options {:identifiers identity
:as-arrays? true
:max-rows max-rows
options {:identifiers identity
:as-arrays? true
:max-rows max-rows
:read-columns (partial sql-jdbc.execute/read-columns driver)
:set-parameters (partial sql-jdbc.execute/set-parameters driver)}]
(with-open [connection (jdbc/get-connection connection)]
......
......@@ -160,7 +160,8 @@
[driver _ expr]
(->date (sql.qp/->honeysql driver expr) (hx/literal "start of year")))
(defmethod driver/date-add :sqlite [driver dt amount unit]
(defmethod driver/date-add :sqlite
[driver dt amount unit]
(let [[multiplier sqlite-unit] (case unit
:second [1 "seconds"]
:minute [1 "minutes"]
......@@ -194,10 +195,12 @@
;;
;; TODO - not sure why this doesn't need to be done in `->honeysql` as well? I think it's because the MBQL date values
;; are funneled through the `date` family of functions above
;; TIMESTAMP FIXME
;;
;; TIMESTAMP FIXME — this doesn't seem like the correct thing to do for non-Dates. I think params only support dates
;; rn however
(s/defmethod sql/->prepared-substitution [:sqlite Temporal] :- sql/PreparedStatementSubstitution
[_ date]
;; for anything that's a Date (usually a java.sql.Timestamp) convert it to a yyyy-MM-dd formatted date literal
;; for anything that's a Temporal value convert it to a yyyy-MM-dd formatted date literal
;; string For whatever reason the SQL generated from parameters ends up looking like `WHERE date(some_field) = ?`
;; sometimes so we need to use just the date rather than a full ISO-8601 string
(sql/make-stmt-subs "?" [(t/format "yyyy-MM-dd" date)]))
......@@ -209,29 +212,46 @@
;; See https://sqlite.org/lang_datefunc.html
;; MEGA HACK
;;
;; if the time portion is zeroed out generate a date() instead, because SQLite isn't smart enough to compare DATEs
;; and DATETIMEs in a way that could be considered to make any sense whatsoever, e.g.
;;
;; date('2019-12-03') < datetime('2019-12-03 00:00')
(defn- zero-time? [t]
(= (t/local-time t) (t/local-time 0)))
(defmethod sql.qp/->honeysql [:sqlite LocalDate]
[_ t]
(hsql/call :date (hx/literal (u.date/format-sql t))))
(defmethod sql.qp/->honeysql [:sqlite LocalDateTime]
[_ t]
(hsql/call :datetime (hx/literal (u.date/format-sql t))))
[driver t]
(if (zero-time? t)
(sql.qp/->honeysql driver (t/local-date t))
(hsql/call :datetime (hx/literal (u.date/format-sql t)))))
(defmethod sql.qp/->honeysql [:sqlite LocalTime]
[_ t]
(hsql/call :time (hx/literal (u.date/format-sql t))))
(defmethod sql.qp/->honeysql [:sqlite OffsetDateTime]
[_ t]
(hsql/call :datetime (hx/literal (u.date/format-sql t))))
[driver t]
(if (zero-time? t)
(sql.qp/->honeysql driver (t/local-date t))
(hsql/call :datetime (hx/literal (u.date/format-sql t)))))
(defmethod sql.qp/->honeysql [:sqlite OffsetTime]
[_ t]
(hsql/call :time (hx/literal (u.date/format-sql t))))
(defmethod sql.qp/->honeysql [:sqlite ZonedDateTime]
[_ t]
(hsql/call :datetime (hx/literal (u.date/format-sql t))))
[driver t]
(println "t:" t) ; NOCOMMIT
(println "(t/local-date t):" (t/local-date t)) ; NOCOMMIT
(if (zero-time? t)
(sql.qp/->honeysql driver (t/local-date t))
(hsql/call :datetime (hx/literal (u.date/format-sql t)))))
;; SQLite `LIKE` clauses are case-insensitive by default, and thus cannot be made case-sensitive. So let people know
;; we have this 'feature' so the frontend doesn't try to present the option to you.
......
(ns metabase.driver.sqlite-test
(:require [expectations :refer [expect]]
[metabase.test.data.datasets :refer [expect-with-driver]]
[metabase.test.util :as tu]))
(:require [clojure.test :refer :all]
[metabase.query-processor-test :as qp.test]
[metabase.test
[data :as data]
[util :as tu]]
[metabase.test.data.datasets :as datasets]))
(expect-with-driver :sqlite
"UTC"
(tu/db-timezone-id))
(deftest timezone-id-test
(datasets/test-driver :sqlite
(is (= "UTC"
(tu/db-timezone-id)))))
(deftest filter-by-date-test
"Make sure filtering against a LocalDate works correctly in SQLite"
(datasets/test-driver :sqlite
(is (= [[225 "2014-03-04T00:00:00Z"]
[409 "2014-03-05T00:00:00Z"]
[917 "2014-03-05T00:00:00Z"]
[995 "2014-03-05T00:00:00Z"]
[159 "2014-03-06T00:00:00Z"]
[951 "2014-03-06T00:00:00Z"]]
(qp.test/rows
(data/run-mbql-query checkins
{:fields [$id $date]
:filter [:and
[:>= $date "2014-03-04"]
[:<= $date "2014-03-06"]]
:order-by [[:asc $date]]}))
(qp.test/rows
(data/run-mbql-query checkins
{:fields [$id $date]
:filter [:between $date "2014-03-04" "2014-03-07"]
:order-by [[:asc $date]]}))))))
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment