Handle ISODate() calls in Mongo native queries [ci drivers]

src/metabase/driver.clj

@@ -312,6 +312,7 @@
              [java.sql.Date                  :type/Date]
              [java.sql.Timestamp             :type/DateTime]
              [java.util.Date                 :type/DateTime]
+             [org.joda.time.DateTime         :type/DateTime]
              [java.util.UUID                 :type/Text]       ; shouldn't this be :type/UUID ?
              [clojure.lang.IPersistentMap    :type/Dictionary]
              [clojure.lang.IPersistentVector :type/Array]

src/metabase/driver/mongo/query_processor.clj

@@ -6,6 +6,7 @@
             [clojure.walk :as walk]
             [cheshire.core :as json]
             (monger [collection :as mc]
+                    joda-time                ; apparently if this is loaded Monger can handle JodaTime dates (?)
                     [operators :refer :all])
             [metabase.driver.mongo.util :refer [with-mongo-connection *mongo-connection* values->base-type]]
             [metabase.models.table :refer [Table]]
@@ -15,9 +16,10 @@
             [metabase.util :as u])
   (:import java.sql.Timestamp
            java.util.Date
-           (com.mongodb CommandResult DB)
            clojure.lang.PersistentArrayMap
+           (com.mongodb CommandResult DB)
            org.bson.types.ObjectId
+           org.joda.time.DateTime
            (metabase.query_processor.interface AgFieldRef
                                                DateTimeField
                                                DateTimeValue
@@ -378,6 +380,35 @@
                     v)}))))
 
 
+;;; ------------------------------------------------------------ Handling ISODate(...) forms ------------------------------------------------------------
+;; In Mongo it's fairly common use ISODate(...) forms in queries, which unfortunately are not valid JSON,
+;; and thus cannot be parsed by Cheshire. But we are clever so we will:
+;;
+;; 1) Convert forms like ISODate(...) to valid JSON forms like ["___ISODate", ...]
+;; 2) Parse Normally
+;; 3) Walk the parsed JSON and convert forms like [:___ISODate ...] to JodaTime dates
+
+(defn- encoded-iso-date? [form]
+  (and (vector? form)
+       (= (first form) "___ISODate")))
+
+(defn- maybe-decode-iso-date-fncall [form]
+  (if (encoded-iso-date? form)
+    (DateTime. (second form))
+    form))
+
+(defn- decode-iso-date-fncalls [query]
+  (walk/postwalk maybe-decode-iso-date-fncall query))
+
+(defn- encode-iso-date-fncalls
+  "Replace occurances of `ISODate(...)` function calls (invalid JSON, but legal in Mongo)
+   with legal JSON forms like `[:___ISODate ...]` that we can decode later."
+  [query-string]
+  (s/replace query-string #"ISODate\(([^)]+)\)" "[\"___ISODate\", $1]"))
+
+
+;;; ------------------------------------------------------------ Query Execution ------------------------------------------------------------
+
 (defn mbql->native
   "Process and run an MBQL query."
   [{database :database, {{source-table-name :name} :source-table} :query, :as query}]
@@ -397,7 +428,7 @@
          (string? collection)
          (map? database)]}
   (let [query   (if (string? query)
-                  (json/parse-string query keyword)
+                  (decode-iso-date-fncalls (json/parse-string (encode-iso-date-fncalls query) keyword))
                   query)
         results (mc/aggregate *mongo-connection* collection query
                               :allow-disk-use true)

test/metabase/driver/mongo_test.clj

@@ -11,8 +11,10 @@
             [metabase.query-processor-test :refer [rows]]
             [metabase.test.data :as data]
             [metabase.test.data.datasets :as datasets]
-            [metabase.test.data.interface :as i])
+            [metabase.test.data.interface :as i]
+            [metabase.test.util :as tu])
   (:import org.bson.types.ObjectId
+           org.joda.time.DateTime
            metabase.driver.mongo.MongoDriver))
 
 ;; ## Logic for selectively running mongo
@@ -173,7 +175,6 @@
 
 
 ;;; Check that we support Mongo BSON ID and can filter by it (#1367)
-
 (i/def-database-definition ^:private with-bson-ids
   ["birds"
    [{:field-name "name", :base-type :type/Text}
@@ -186,3 +187,27 @@
   (rows (data/dataset metabase.driver.mongo-test/with-bson-ids
           (data/run-query birds
             (ql/filter (ql/= $bird_id "abcdefabcdefabcdefabcdef"))))))
+
+
+;;; ------------------------------------------------------------ Test that we can handle native queries with "ISODate(...)" forms (#3741) ------------------------------------------------------------
+(tu/resolve-private-vars metabase.driver.mongo.query-processor
+  maybe-decode-iso-date-fncall decode-iso-date-fncalls encode-iso-date-fncalls)
+
+(expect
+  "[{\"$match\":{\"date\":{\"$gte\":[\"___ISODate\", \"2012-01-01\"]}}}]"
+  (encode-iso-date-fncalls "[{\"$match\":{\"date\":{\"$gte\":ISODate(\"2012-01-01\")}}}]"))
+
+(expect
+  (DateTime. "2012-01-01")
+  (maybe-decode-iso-date-fncall ["___ISODate" "2012-01-01"]))
+
+(expect
+  [{:$match {:date {:$gte (DateTime. "2012-01-01")}}}]
+  (decode-iso-date-fncalls [{:$match {:date {:$gte ["___ISODate" "2012-01-01"]}}}]))
+
+(datasets/expect-with-engine :mongo
+  5
+  (count (rows (qp/process-query {:native   {:query      "[{\"$match\": {\"date\": {\"$gte\": ISODate(\"2015-12-20\")}}}]"
+                                             :collection "checkins"}
+                                  :type     :native
+                                  :database (:id (mongo-db))}))))