From 113c05589ba920ab2f6f53a3269491fce3c93f18 Mon Sep 17 00:00:00 2001
From: lbrdnk <lbrdnk@users.noreply.github.com>
Date: Tue, 6 Feb 2024 12:10:52 +0100
Subject: [PATCH] Mongo java driver upgrade (#38017)
* tmp: patched monger for 4.11.1 mongo java driver
* tmp: Update monger utils
Aggregation probably wont work now, but we are not using those from monger anyway. With this change in place I'm able to load needed namespaces and create test-data dataset successfully.
Commit contains lot of condo errors that should be resolved while porting.
* WIP: Monger removed in favor of java driver wrapper
* Update java driver wrapper
* Update srv test
* Update comments
* Use non keywordized run-command for `dbms-version`
* Fix according to e2e tests
* Cleanup
* Separate `java-driver-wrapper` into multiple namespaces
* Fix semantic type inference for serialized json
* Fix options for run-command
* Cleanup json namespace
* Cleanup conversion namespace
* Cleanup operators
* Update kondo in operators ns
* Cleanup connection namespace
* Cleanup mongo namespace
* Cleanup util namespace
* Add todo
* Move session related code to util
* Cleanup database namespace
* Update docstring for conn string generation
* Update docstrings
* Update tests
* Update linter for with macros
* Update modules/drivers/mongo/src/metabase/driver/mongo/connection.clj
Co-authored-by: metamben <103100869+metamben@users.noreply.github.com>
* Update modules/drivers/mongo/src/metabase/driver/mongo/connection.clj
Co-authored-by: metamben <103100869+metamben@users.noreply.github.com>
* Use transient in from-document for building a map
Co-authored-by: metamben <103100869+metamben@users.noreply.github.com>
* Update can-connect to use let form
- Avoid nested `with-mongo-database` call.
- Avoid creating a set used for searching for database name.
* Remove redundant set use from describe-database
* Change from-document keywordize default to false
* Remove log message translation
* Update maybe-add-ssl-context-to-builder! to always return builder
* Indent from-document
* Remove redundant ConvertToDocument extensions
* Use oredered-map in do-find's sort
* Use ex-info in details-normalized
* Add imports and update comment in execute ns
* Update fixme comment
* Pass opts instead of a selection to from-document in do-find
* Avoid unnecessary double dot call
* Update connection test ns according to review remarks
* Make tests parallel
* Docstring update
* Update docstring
---------
Co-authored-by: metamben <103100869+metamben@users.noreply.github.com>
---
.clj-kondo/config.edn | 3 +-
modules/drivers/mongo/deps.edn | 3 +-
.../mongo/src/metabase/driver/mongo.clj | 170 +++------
.../src/metabase/driver/mongo/connection.clj | 120 ++++++
.../src/metabase/driver/mongo/conversion.clj | 127 +++++++
.../src/metabase/driver/mongo/database.clj | 62 ++++
.../src/metabase/driver/mongo/execute.clj | 54 +--
.../mongo/src/metabase/driver/mongo/json.clj | 20 +
.../src/metabase/driver/mongo/operators.clj | 216 +++++++++++
.../metabase/driver/mongo/query_processor.clj | 22 +-
.../mongo/src/metabase/driver/mongo/util.clj | 346 +++++-------------
.../metabase/driver/mongo/connection_test.clj | 141 +++++++
.../metabase/driver/mongo/conversion_test.clj | 28 ++
.../metabase/driver/mongo/database_test.clj | 19 +
.../metabase/driver/mongo/execute_test.clj | 5 +-
.../test/metabase/driver/mongo/util_test.clj | 172 ---------
.../mongo/test/metabase/driver/mongo_test.clj | 79 ++--
.../mongo/test/metabase/test/data/mongo.clj | 25 +-
src/metabase/driver/util.clj | 13 +-
19 files changed, 981 insertions(+), 644 deletions(-)
create mode 100644 modules/drivers/mongo/src/metabase/driver/mongo/connection.clj
create mode 100644 modules/drivers/mongo/src/metabase/driver/mongo/conversion.clj
create mode 100644 modules/drivers/mongo/src/metabase/driver/mongo/database.clj
create mode 100644 modules/drivers/mongo/src/metabase/driver/mongo/json.clj
create mode 100644 modules/drivers/mongo/src/metabase/driver/mongo/operators.clj
create mode 100644 modules/drivers/mongo/test/metabase/driver/mongo/connection_test.clj
create mode 100644 modules/drivers/mongo/test/metabase/driver/mongo/conversion_test.clj
create mode 100644 modules/drivers/mongo/test/metabase/driver/mongo/database_test.clj
delete mode 100644 modules/drivers/mongo/test/metabase/driver/mongo/util_test.clj
diff --git a/.clj-kondo/config.edn b/.clj-kondo/config.edn
index 2d64cd7b482..9daf44e2713 100644
--- a/.clj-kondo/config.edn
+++ b/.clj-kondo/config.edn
@@ -432,7 +432,8 @@
metabase.db.schema-migrations-test.impl/with-temp-empty-app-db clojure.core/let
metabase.domain-entities.malli/defn schema.core/defn
metabase.driver.mongo.query-processor/mongo-let clojure.core/let
- metabase.driver.mongo.util/with-mongo-connection clojure.core/let
+ metabase.driver.mongo.connection/with-mongo-client clojure.core/let
+ metabase.driver.mongo.connection/with-mongo-database clojure.core/let
metabase.driver.sql-jdbc.actions/with-jdbc-transaction clojure.core/let
metabase.driver.sql-jdbc.connection/with-connection-spec-for-testing-connection clojure.core/let
metabase.driver.sql-jdbc.execute.diagnostic/capturing-diagnostic-info clojure.core/fn
diff --git a/modules/drivers/mongo/deps.edn b/modules/drivers/mongo/deps.edn
index 1b494f6cc0b..452588e1ecf 100644
--- a/modules/drivers/mongo/deps.edn
+++ b/modules/drivers/mongo/deps.edn
@@ -3,5 +3,4 @@
:deps
{com.google.guava/guava {:mvn/version "32.1.3-jre"}
- com.novemberain/monger {:mvn/version "3.6.0"
- :exclusions [com.google.guava/guava]}}}
+ org.mongodb/mongodb-driver-sync {:mvn/version "4.11.1"}}}
diff --git a/modules/drivers/mongo/src/metabase/driver/mongo.clj b/modules/drivers/mongo/src/metabase/driver/mongo.clj
index 443502f155a..3620223fac2 100644
--- a/modules/drivers/mongo/src/metabase/driver/mongo.clj
+++ b/modules/drivers/mongo/src/metabase/driver/mongo.clj
@@ -5,37 +5,30 @@
[cheshire.generate :as json.generate]
[clojure.string :as str]
[flatland.ordered.map :as ordered-map]
- [java-time.api :as t]
[metabase.db.metadata-queries :as metadata-queries]
[metabase.driver :as driver]
[metabase.driver.common :as driver.common]
+ [metabase.driver.mongo.connection :as mongo.connection]
+ [metabase.driver.mongo.database :as mongo.db]
[metabase.driver.mongo.execute :as mongo.execute]
+ [metabase.driver.mongo.json]
[metabase.driver.mongo.parameters :as mongo.params]
[metabase.driver.mongo.query-processor :as mongo.qp]
- [metabase.driver.mongo.util :refer [with-mongo-connection] :as mongo.util]
+ [metabase.driver.mongo.util :as mongo.util]
[metabase.driver.util :as driver.u]
[metabase.lib.metadata :as lib.metadata]
[metabase.lib.metadata.protocols :as lib.metadata.protocols]
[metabase.query-processor.store :as qp.store]
- [metabase.query-processor.timezone :as qp.timezone]
[metabase.util :as u]
[metabase.util.log :as log]
- [monger.command :as cmd]
- [monger.conversion :as m.conversion]
- [monger.core :as mg]
- [monger.db :as mdb]
- [monger.json]
[taoensso.nippy :as nippy])
(:import
- (com.mongodb DB DBObject)
- (java.time Instant LocalDate LocalDateTime LocalTime OffsetDateTime OffsetTime ZonedDateTime)
- (org.bson.types ObjectId)))
+ (org.bson.types ObjectId)
+ (com.mongodb.client MongoClient MongoDatabase)))
(set! *warn-on-reflection* true)
-;; See http://clojuremongodb.info/articles/integration.html Loading this namespace will load appropriate Monger
-;; integrations with Cheshire.
-(comment monger.json/keep-me)
+(comment metabase.driver.mongo.json/keep-me)
;; JSON Encoding (etc.)
@@ -53,18 +46,18 @@
(driver/register! :mongo)
(defmethod driver/can-connect? :mongo
- [_ details]
- (with-mongo-connection [^DB conn, details]
- (let [db-stats (-> (cmd/db-stats conn)
- (m.conversion/from-db-object :keywordize))
- db-names (mg/get-db-names mongo.util/*mongo-client*)]
+ [_ db-details]
+ (mongo.connection/with-mongo-client [^MongoClient c db-details]
+ (let [db-names (mongo.util/list-database-names c)
+ db (mongo.util/database c (mongo.db/db-name db-details))
+ db-stats (mongo.util/run-command db {:dbStats 1} :keywordize true)]
(and
;; 1. check db.dbStats command completes successfully
(= (float (:ok db-stats))
1.0)
;; 2. check the database is actually on the server
;; (this is required because (1) is true even if the database doesn't exist)
- (contains? db-names (:db db-stats))))))
+ (boolean (some #(= % (:db db-stats)) db-names))))))
(defmethod driver/humanize-connection-error-message
:mongo
@@ -106,7 +99,7 @@
(defmethod driver/sync-in-context :mongo
[_ database do-sync-fn]
- (with-mongo-connection [_ database]
+ (mongo.connection/with-mongo-client [_ database]
(do-sync-fn)))
(defn- val->semantic-type [field-value]
@@ -118,8 +111,8 @@
;; 2. json?
(and (string? field-value)
- (or (str/starts-with? "{" field-value)
- (str/starts-with? "[" field-value)))
+ (or (str/starts-with? field-value "{")
+ (str/starts-with? field-value "[")))
(when-let [j (u/ignore-exceptions (json/parse-string field-value))]
(when (or (map? j)
(sequential? j))
@@ -188,8 +181,8 @@
(defmethod driver/dbms-version :mongo
[_driver database]
- (with-mongo-connection [^com.mongodb.DB conn database]
- (let [build-info (mg/command conn {:buildInfo 1})
+ (mongo.connection/with-mongo-database [db database]
+ (let [build-info (mongo.util/run-command db {:buildInfo 1})
version-array (get build-info "versionArray")
sanitized-version-array (into [] (take-while nat-int?) version-array)]
(when (not= (take 3 version-array) (take 3 sanitized-version-array))
@@ -200,57 +193,38 @@
(defmethod driver/describe-database :mongo
[_ database]
- (with-mongo-connection [^com.mongodb.DB conn database]
- {:tables (set (for [collection (disj (mdb/get-collection-names conn) "system.indexes")]
+ (mongo.connection/with-mongo-database [^MongoDatabase db database]
+ {:tables (set (for [collection (mongo.util/list-collection-names db)
+ :when (not= collection "system.indexes")]
{:schema nil, :name collection}))}))
(defmethod driver/describe-table-indexes :mongo
[_ database table]
- (with-mongo-connection [^com.mongodb.DB conn database]
- ;; using raw DBObject instead of calling `monger/indexes-on`
- ;; because in case a compound index has more than 8 keys, the `key` returned by
- ;;`monger/indexes-on` will be a hash-map, and with a hash map we can't determine
- ;; which key is the first key.
- (->> (.getIndexInfo (.getCollection conn (:name table)))
- (map (fn [index]
+ (mongo.connection/with-mongo-database [^MongoDatabase db database]
+ (let [collection (mongo.util/collection db (:name table))]
+ (->> (mongo.util/list-indexes collection)
+ (map (fn [index]
;; for text indexes, column names are specified in the weights
- (if (contains? index "textIndexVersion")
- (get index "weights")
- (get index "key"))))
- (map (comp name first keys))
- ;; mongo support multi key index, aka nested fields index, so we need to split the keys
- ;; and represent it as a list of field names
- (map #(if (str/includes? % ".")
- {:type :nested-column-index
- :value (str/split % #"\.")}
- {:type :normal-column-index
- :value %}))
- set)))
-
-(defn- from-db-object
- "This is mostly a copy of the monger library's own function of the same name with the
- only difference that it uses an ordered map to represent the document. This ensures that
- the order of the top level fields of the table is preserved. For anything that's not a
- DBObject, it falls back to the original function."
- [input]
- (if (instance? DBObject input)
- (let [^DBObject dbobj input]
- (reduce (fn [m ^String k]
- (assoc m (keyword k) (m.conversion/from-db-object (.get dbobj k) true)))
- (ordered-map/ordered-map)
- (.keySet dbobj)))
- (m.conversion/from-db-object input true)))
-
-(defn- sample-documents [^com.mongodb.DB conn table sort-direction]
- (let [collection (.getCollection conn (:name table))]
- (with-open [cursor (doto (.find collection
- (m.conversion/to-db-object {})
- (m.conversion/as-field-selector []))
- (.limit metadata-queries/nested-field-sample-limit)
- (.skip 0)
- (.sort (m.conversion/to-db-object {:_id sort-direction}))
- (.batchSize 256))]
- (map from-db-object cursor))))
+ (if (contains? index "textIndexVersion")
+ (get index "weights")
+ (get index "key"))))
+ (map (comp name first keys))
+ ;; mongo support multi key index, aka nested fields index, so we need to split the keys
+ ;; and represent it as a list of field names
+ (map #(if (str/includes? % ".")
+ {:type :nested-column-index
+ :value (str/split % #"\.")}
+ {:type :normal-column-index
+ :value %}))
+ set))))
+
+(defn- sample-documents [^MongoDatabase db table sort-direction]
+ (let [coll (mongo.util/collection db (:name table))]
+ (mongo.util/do-find coll {:keywordize true
+ :limit metadata-queries/nested-field-sample-limit
+ :skip 0
+ :sort-criteria [[:_id sort-direction]]
+ :batch-size 256})))
(defn- table-sample-column-info
"Sample the rows (i.e., documents) in `table` and return a map of information about the column keys we found in that
@@ -258,7 +232,7 @@
{:_id {:count 200, :len nil, :types {java.lang.Long 200}, :semantic-types nil, :nested-fields nil},
:severity {:count 200, :len nil, :types {java.lang.Long 200}, :semantic-types nil, :nested-fields nil}}"
- [^com.mongodb.DB conn, table]
+ [^MongoDatabase db table]
(try
(reduce
(fn [field-defs row]
@@ -267,14 +241,14 @@
fields
(recur more-keys (update fields k (partial update-field-attrs (k row)))))))
(ordered-map/ordered-map)
- (concat (sample-documents conn table 1) (sample-documents conn table -1)))
+ (concat (sample-documents db table 1) (sample-documents db table -1)))
(catch Throwable t
(log/error (format "Error introspecting collection: %s" (:name table)) t))))
(defmethod driver/describe-table :mongo
[_ database table]
- (with-mongo-connection [^com.mongodb.DB conn database]
- (let [column-info (table-sample-column-info conn table)]
+ (mongo.connection/with-mongo-database [^MongoDatabase db database]
+ (let [column-info (table-sample-column-info db table)]
{:schema nil
:name (:name table)
:fields (first
@@ -342,53 +316,13 @@
(defmethod driver/execute-reducible-query :mongo
[_ query context respond]
- (with-mongo-connection [_ (lib.metadata/database (qp.store/metadata-provider))]
+ (mongo.connection/with-mongo-client [_ (lib.metadata/database (qp.store/metadata-provider))]
(mongo.execute/execute-reducible-query query context respond)))
(defmethod driver/substitute-native-parameters :mongo
[driver inner-query]
(mongo.params/substitute-native-parameters driver inner-query))
-;; It seems to be the case that the only thing BSON supports is DateTime which is basically the equivalent of Instant;
-;; for the rest of the types, we'll have to fake it
-(extend-protocol m.conversion/ConvertToDBObject
- Instant
- (to-db-object [t]
- (org.bson.BsonDateTime. (t/to-millis-from-epoch t)))
-
- LocalDate
- (to-db-object [t]
- (m.conversion/to-db-object (t/local-date-time t (t/local-time 0))))
-
- LocalDateTime
- (to-db-object [t]
- ;; QP store won't be bound when loading test data for example.
- (m.conversion/to-db-object (t/instant t (t/zone-id (try
- (qp.timezone/results-timezone-id)
- (catch Throwable _
- "UTC"))))))
-
- LocalTime
- (to-db-object [t]
- (m.conversion/to-db-object (t/local-date-time (t/local-date "1970-01-01") t)))
-
- OffsetDateTime
- (to-db-object [t]
- (m.conversion/to-db-object (t/instant t)))
-
- OffsetTime
- (to-db-object [t]
- (m.conversion/to-db-object (t/offset-date-time (t/local-date "1970-01-01") t (t/zone-offset t))))
-
- ZonedDateTime
- (to-db-object [t]
- (m.conversion/to-db-object (t/instant t))))
-
-(extend-protocol m.conversion/ConvertFromDBObject
- java.util.Date
- (from-db-object [t _]
- (t/instant t)))
-
(defmethod driver/db-start-of-week :mongo
[_]
:sunday)
@@ -406,7 +340,9 @@
:order-by [[:desc [:field (get-id-field-id table) nil]]]}]
(metadata-queries/table-rows-sample table fields rff (merge mongo-opts opts)))))
-(comment
+;; Following code is using monger. Leaving it here for a reference as it could be transformed when there is need
+;; for ssl experiments.
+#_(comment
(require '[clojure.java.io :as io]
'[monger.credentials :as mcred])
(import javax.net.ssl.SSLSocketFactory)
diff --git a/modules/drivers/mongo/src/metabase/driver/mongo/connection.clj b/modules/drivers/mongo/src/metabase/driver/mongo/connection.clj
new file mode 100644
index 00000000000..0b0d2a4b438
--- /dev/null
+++ b/modules/drivers/mongo/src/metabase/driver/mongo/connection.clj
@@ -0,0 +1,120 @@
+(ns metabase.driver.mongo.connection
+ "This namespace contains code responsible for connecting to mongo deployment."
+ (:require
+ [clojure.string :as str]
+ [metabase.config :as config]
+ [metabase.driver.mongo.database :as mongo.db]
+ [metabase.driver.mongo.util :as mongo.util]
+ [metabase.driver.util :as driver.u]
+ [metabase.util :as u]
+ [metabase.util.log :as log]
+ [metabase.util.ssh :as ssh])
+ (:import
+ (com.mongodb ConnectionString MongoClientSettings MongoClientSettings$Builder)
+ (com.mongodb.connection SslSettings$Builder)))
+
+(set! *warn-on-reflection* true)
+
+(def ^:dynamic *mongo-client*
+ "Stores an instance of `MongoClient` bound by [[with-mongo-client]]."
+ nil)
+
+(defn db-details->connection-string
+ "Generate connection string from database details.
+
+ - `?authSource` is always prestent because we are using `dbname`.
+ - We let the user override options we are passing in by means of `additional-options`."
+ [{:keys [use-conn-uri conn-uri host port user authdb pass dbname additional-options use-srv ssl] :as _db-details}]
+ ;; Connection string docs:
+ ;; http://mongodb.github.io/mongo-java-driver/4.11/apidocs/mongodb-driver-core/com/mongodb/ConnectionString.html
+ (if use-conn-uri
+ conn-uri
+ (str
+ (if use-srv "mongodb+srv" "mongodb")
+ "://"
+ (when (seq user) (str user (when (seq pass) (str ":" pass)) "@"))
+ host
+ (when (and (not use-srv) (some? port)) (str ":" port))
+ "/"
+ dbname
+ "?authSource=" (if (empty? authdb) "admin" authdb)
+ "&appName=" config/mb-app-id-string
+ "&connectTimeoutMS=" (driver.u/db-connection-timeout-ms)
+ "&serverSelectionTimeoutMS=" (driver.u/db-connection-timeout-ms)
+ (when ssl "&ssl=true")
+ (when (seq additional-options) (str "&" additional-options)))))
+
+(defn- maybe-add-ssl-context-to-builder!
+ "Add SSL context to `builder` using `_db-details`. Mutates and returns `builder`."
+ [^MongoClientSettings$Builder builder
+ {:keys [ssl-cert ssl-use-client-auth client-ssl-cert client-ssl-key] :as _db-details}]
+ (let [server-cert? (not (str/blank? ssl-cert))
+ client-cert? (and ssl-use-client-auth
+ (not-any? str/blank? [client-ssl-cert client-ssl-key]))]
+ (if (or client-cert? server-cert?)
+ (let [ssl-params (cond-> {}
+ server-cert? (assoc :trust-cert ssl-cert)
+ client-cert? (assoc :private-key client-ssl-key
+ :own-cert client-ssl-cert))
+ ssl-context (driver.u/ssl-context ssl-params)]
+ (.applyToSslSettings builder
+ (reify com.mongodb.Block
+ (apply [_this builder]
+ (.context ^SslSettings$Builder builder ssl-context)))))
+ builder)))
+
+(defn db-details->mongo-client-settings
+ "Generate `MongoClientSettings` from `db-details`. `ConnectionString` is generated and applied to
+ `MongoClientSettings$Builder` first. Then ssl context is udated in the `builder` object.
+ Afterwards, `MongoClientSettings` are built using `.build`."
+ ^MongoClientSettings
+ [{:keys [use-conn-uri ssl] :as db-details}]
+ (let [connection-string (-> db-details
+ db-details->connection-string
+ ConnectionString.)
+ builder (com.mongodb.MongoClientSettings/builder)]
+ (.applyConnectionString builder connection-string)
+ (when (and ssl (not use-conn-uri))
+ (maybe-add-ssl-context-to-builder! builder db-details))
+ (.build builder)))
+
+(defn do-with-mongo-client
+ "Implementation of [[with-mongo-client]]."
+ [thunk database]
+ (let [db-details (mongo.db/details-normalized database)]
+ (ssh/with-ssh-tunnel [details-with-tunnel db-details]
+ (let [client (mongo.util/mongo-client (db-details->mongo-client-settings details-with-tunnel))]
+ (log/debug (u/format-color 'cyan "Opened new MongoClient."))
+ (try
+ (binding [*mongo-client* client]
+ (thunk client))
+ (finally
+ (mongo.util/close client)
+ (log/debug (u/format-color 'cyan "Closed MongoClient."))))))))
+
+(defmacro with-mongo-client
+ "Create instance of `MongoClient` for `database` and bind it to [[*mongo-client*]]. `database` can be anything
+ digestable by [[mongo.db/details-normalized]]. Call of this macro in its body will reuse existing
+ [[*mongo-client*]]."
+ {:clj-kondo/lint-as 'clojure.core/let
+ :clj-kondo/ignore [:unresolved-symbol :type-mismatch]}
+ [[client-sym database] & body]
+ `(let [f# (fn [~client-sym] ~@body)]
+ (if (nil? *mongo-client*)
+ (do-with-mongo-client f# ~database)
+ (f# *mongo-client*))))
+
+(defn do-with-mongo-database
+ "Implementation of [[with-mongo-database]]."
+ [thunk database]
+ (let [db-name (-> database mongo.db/details-normalized mongo.db/details->db-name)]
+ (with-mongo-client [c database]
+ (thunk (mongo.util/database c db-name)))))
+
+(defmacro with-mongo-database
+ "Utility for accessing database directly instead of a client. For more info see [[with-mongo-client]]."
+ {:clj-kondo/lint-as 'clojure.core/let
+ :clj-kondo/ignore [:unresolved-symbol :type-mismatch]}
+ [[db-sym database] & body]
+ `(let [f# (fn [~db-sym] ~@body)]
+ (do-with-mongo-database f# ~database)))
diff --git a/modules/drivers/mongo/src/metabase/driver/mongo/conversion.clj b/modules/drivers/mongo/src/metabase/driver/mongo/conversion.clj
new file mode 100644
index 00000000000..5a70474f1c6
--- /dev/null
+++ b/modules/drivers/mongo/src/metabase/driver/mongo/conversion.clj
@@ -0,0 +1,127 @@
+(ns metabase.driver.mongo.conversion
+ "This namespace contains utilities for conversion between mongo specific types and clojure.
+
+ It is copy of monger's conversion namespace, that was adjusted for the needs of Document. Extensions that were
+ previously implemented in our mongo driver were also moved into this namespace.
+
+ [[to-document]] and [[from-document]] are meant to be used for transformation of clojure data into mongo aggregation
+ pipelines and results back into clojure structures.
+
+ TODO: Logic is copied from monger's conversions. It seems that lot of implementations are redundant. I'm not sure
+ yet. We should consider further simplifying the namespace.
+ TODO: Consider use of bson's encoders/decoders/codecs instead this code.
+ TODO: Or consider adding types from org.bson package -- eg. BsonInt32. If we'd decide to go this way, we could
+ transform ejson completely to clojure structures. That however requires deciding how to represent
+ eg. ObjectIds (it could be eg. {$oid \"...\"} which would copy EJSON v2 way). [EJSON v2 doc](https://www.mongodb.com/docs/manual/reference/mongodb-extended-json/).
+ TODO: Names of protocol functions and protocols are bit misleading as were in monger.
+
+ TODOs should be addressed during follow-up of monger removal."
+ (:require
+ [flatland.ordered.map :as ordered-map]
+ [java-time.api :as t]
+ [metabase.query-processor.timezone :as qp.timezone]))
+
+(set! *warn-on-reflection* true)
+
+;;;; Protocols defined originally in monger, adjusted for `Document` follow.
+
+(defprotocol ConvertFromDocument
+ (from-document [input opts] "Converts given DBObject instance to a piece of Clojure data"))
+
+(extend-protocol ConvertFromDocument
+ nil
+ (from-document [input _opts] input)
+
+ Object
+ (from-document [input _opts] input)
+
+ org.bson.types.Decimal128
+ (from-document [^org.bson.types.Decimal128 input _opts]
+ (.bigDecimalValue input))
+
+ java.util.List
+ (from-document [^java.util.List input opts]
+ (mapv #(from-document % opts) input))
+
+ java.util.Date
+ (from-document [t _]
+ (t/instant t))
+
+ org.bson.Document
+ (from-document [input {:keys [keywordize] :or {keywordize false} :as opts}]
+ (persistent! (reduce (if keywordize
+ (fn [m ^String k]
+ (assoc! m (keyword k) (from-document (.get input k) opts)))
+ (fn [m ^String k]
+ (assoc! m k (from-document (.get input k) opts))))
+ (transient (ordered-map/ordered-map))
+ (.keySet input)))))
+
+(defprotocol ConvertToDocument
+ (to-document [input] "Converts given piece of Clojure data to org.bson.Document usable by java driver."))
+
+(extend-protocol ConvertToDocument
+ nil
+ (to-document [_input] nil)
+
+ clojure.lang.Ratio
+ (to-document [input] (double input))
+
+ clojure.lang.Keyword
+ (to-document [input] (.getName input))
+
+ clojure.lang.Named
+ (to-document [input] (.getName input))
+
+ clojure.lang.IPersistentMap
+ (to-document [input]
+ (let [o (org.bson.Document.)]
+ (doseq [[k v] input]
+ (.put o (to-document k) (to-document v)))
+ o))
+
+ java.util.List
+ (to-document [input] (mapv to-document input))
+
+ java.util.Set
+ (to-document [input] (mapv to-document input))
+
+ Object
+ (to-document [input] input))
+
+;;;; Protocol extensions gathered from our mongo driver's code follow.
+
+;; It seems to be the case that the only thing BSON supports is DateTime which is basically the equivalent
+;; of Instant; for the rest of the types, we'll have to fake it. (Cam)
+(extend-protocol ConvertToDocument
+ java.time.Instant
+ (to-document [t]
+ (org.bson.BsonDateTime. (t/to-millis-from-epoch t)))
+
+ java.time.LocalDate
+ (to-document [t]
+ (to-document (t/local-date-time t (t/local-time 0))))
+
+ java.time.LocalDateTime
+ (to-document [t]
+ ;; QP store won't be bound when loading test data for example.
+ (to-document (t/instant t (t/zone-id (try
+ (qp.timezone/results-timezone-id)
+ (catch Throwable _
+ "UTC"))))))
+
+ java.time.LocalTime
+ (to-document [t]
+ (to-document (t/local-date-time (t/local-date "1970-01-01") t)))
+
+ java.time.OffsetDateTime
+ (to-document [t]
+ (to-document (t/instant t)))
+
+ java.time.OffsetTime
+ (to-document [t]
+ (to-document (t/offset-date-time (t/local-date "1970-01-01") t (t/zone-offset t))))
+
+ java.time.ZonedDateTime
+ (to-document [t]
+ (to-document (t/instant t))))
diff --git a/modules/drivers/mongo/src/metabase/driver/mongo/database.clj b/modules/drivers/mongo/src/metabase/driver/mongo/database.clj
new file mode 100644
index 00000000000..6e0828150a9
--- /dev/null
+++ b/modules/drivers/mongo/src/metabase/driver/mongo/database.clj
@@ -0,0 +1,62 @@
+(ns metabase.driver.mongo.database
+ "This namespace contains functions for work with mongo specific database and database details."
+ (:require
+ [metabase.lib.metadata.protocols :as lib.metadata.protocols]
+ [metabase.models.secret :as secret]
+ [metabase.query-processor.store :as qp.store]
+ [metabase.util.i18n :refer [tru]])
+ (:import
+ (com.mongodb ConnectionString)))
+
+(set! *warn-on-reflection* true)
+
+(defn- fqdn?
+ "A very simple way to check if a hostname is fully-qualified:
+ Check if there are two or more periods in the name."
+ [host]
+ (<= 2 (-> host frequencies (get \. 0))))
+
+(defn- validate-db-details! [{:keys [use-conn-uri conn-uri use-srv host] :as _db-details}]
+ (when (and use-srv (not (fqdn? host)))
+ (throw (ex-info (tru "Using DNS SRV requires a FQDN for host")
+ {:host host})))
+ (when (and use-conn-uri (empty? (-> (ConnectionString. conn-uri) .getDatabase)))
+ (throw (ex-info (tru "No database name specified in URI.")
+ {:host host}))))
+
+(defn- update-ssl-db-details
+ [db-details]
+ (-> db-details
+ (assoc :client-ssl-key (secret/get-secret-string db-details "client-ssl-key"))
+ (dissoc :client-ssl-key-creator-id
+ :client-ssl-key-created-at
+ :client-ssl-key-id
+ :client-ssl-key-source)))
+
+(defn details-normalized
+ "Gets db-details for `database`. Details are then validated and ssl related keys are updated."
+ [database]
+ (let [db-details
+ (cond
+ (integer? database) (qp.store/with-metadata-provider database
+ (:details (lib.metadata.protocols/database (qp.store/metadata-provider))))
+ (string? database) {:dbname database}
+ (:dbname (:details database)) (:details database) ; entire Database obj
+ (:dbname database) database ; connection details map only
+ (:conn-uri database) database ; connection URI has all the parameters
+ (:conn-uri (:details database)) (:details database)
+ :else
+ (throw (ex-info (tru "Unable to to get database details.")
+ {:database database})))]
+ (validate-db-details! db-details)
+ (update-ssl-db-details db-details)))
+
+(defn details->db-name
+ "Get database name from database `:details`."
+ ^String [{:keys [dbname conn-uri] :as _db-details}]
+ (or (not-empty dbname) (-> (com.mongodb.ConnectionString. conn-uri) .getDatabase)))
+
+(defn db-name
+ "Get db-name from `database`. `database` value is something normalizable to database details."
+ [database]
+ (-> database details-normalized details->db-name))
diff --git a/modules/drivers/mongo/src/metabase/driver/mongo/execute.clj b/modules/drivers/mongo/src/metabase/driver/mongo/execute.clj
index 376f15a64fe..86cf1225abd 100644
--- a/modules/drivers/mongo/src/metabase/driver/mongo/execute.clj
+++ b/modules/drivers/mongo/src/metabase/driver/mongo/execute.clj
@@ -3,19 +3,22 @@
[clojure.core.async :as a]
[clojure.set :as set]
[clojure.string :as str]
+ [metabase.driver.mongo.connection :as mongo.connection]
+ [metabase.driver.mongo.conversion :as mongo.conversion]
+ [metabase.driver.mongo.database :as mongo.db]
[metabase.driver.mongo.query-processor :as mongo.qp]
[metabase.driver.mongo.util :as mongo.util]
+ [metabase.lib.metadata :as lib.metadata]
[metabase.query-processor.context :as qp.context]
[metabase.query-processor.error-type :as qp.error-type]
[metabase.query-processor.reducible :as qp.reducible]
+ [metabase.query-processor.store :as qp.store]
[metabase.util.i18n :refer [tru]]
[metabase.util.log :as log]
- [metabase.util.malli :as mu]
- [monger.conversion :as m.conversion]
- [monger.util :as m.util])
+ [metabase.util.malli :as mu])
(:import
- (com.mongodb BasicDBObject DB DBObject)
- (com.mongodb.client AggregateIterable ClientSession MongoDatabase MongoCursor)
+ (com.mongodb.client AggregateIterable ClientSession MongoCursor MongoDatabase)
+ (java.util ArrayList Collection)
(java.util.concurrent TimeUnit)
(org.bson BsonBoolean BsonInt32)))
@@ -105,16 +108,16 @@
;;; ------------------------------------------------------ Rows ------------------------------------------------------
(defn- row->vec [row-col-names]
- (fn [^DBObject row]
+ (fn [^org.bson.Document row]
(mapv (fn [col-name]
(let [col-parts (str/split col-name #"\.")
val (reduce
- (fn [^BasicDBObject object ^String part-name]
+ (fn [^org.bson.Document object ^String part-name]
(when object
(.get object part-name)))
row
col-parts)]
- (m.conversion/from-db-object val :keywordize)))
+ (mongo.conversion/from-document val {:keywordize true})))
row-col-names)))
(defn- post-process-row [row-col-names]
@@ -126,7 +129,7 @@
;;; | Run |
;;; +----------------------------------------------------------------------------------------------------------------+
-(defn- row-keys [^DBObject row]
+(defn- row-keys [^org.bson.Document row]
(when row
(.keySet row)))
@@ -146,8 +149,8 @@
^ClientSession session
stages timeout-ms]
(let [coll (.getCollection db coll)
- pipe (m.util/into-array-list (m.conversion/to-db-object stages))
- aggregate (.aggregate coll session pipe BasicDBObject)]
+ pipe (ArrayList. ^Collection (mongo.conversion/to-document stages))
+ aggregate (.aggregate coll session pipe)]
(init-aggregate! aggregate timeout-ms)))
(defn- reducible-rows [context ^MongoCursor cursor first-row post-process]
@@ -176,36 +179,19 @@
[]
(reducible-rows context cursor first-row (post-process-row row-col-names))))))
-
-(defn- connection->database
- ^MongoDatabase
- [^DB connection]
- (let [db-name (.getName connection)]
- (.. connection getMongoClient (getDatabase db-name))))
-
-(defn- start-session!
- ^ClientSession
- [^DB connection]
- (.. connection getMongoClient startSession))
-
-(defn- kill-session!
- [^MongoDatabase db
- ^ClientSession session]
- (let [session-id (.. session getServerSession getIdentifier)
- kill-cmd (BasicDBObject. "killSessions" [session-id])]
- (.runCommand db kill-cmd)))
-
(defn execute-reducible-query
- "Process and run a native MongoDB query."
+ "Process and run a native MongoDB query. This function expects initialized [[mongo.connection/*mongo-client*]]."
[{{query :query collection-name :collection :as native-query} :native} context respond]
{:pre [(string? collection-name) (fn? respond)]}
(let [query (cond-> query
(string? query) mongo.qp/parse-query-string)
- client-database (connection->database mongo.util/*mongo-connection*)]
- (with-open [session ^ClientSession (start-session! mongo.util/*mongo-connection*)]
+ database (lib.metadata/database (qp.store/metadata-provider))
+ db-name (mongo.db/db-name database)
+ client-database (mongo.util/database mongo.connection/*mongo-client* db-name)]
+ (with-open [session ^ClientSession (mongo.util/start-session! mongo.connection/*mongo-client*)]
(a/go
(when (a/<! (qp.context/canceled-chan context))
- (kill-session! client-database session)))
+ (mongo.util/kill-session! client-database session)))
(let [aggregate ^AggregateIterable (*aggregate* client-database
collection-name
session
diff --git a/modules/drivers/mongo/src/metabase/driver/mongo/json.clj b/modules/drivers/mongo/src/metabase/driver/mongo/json.clj
new file mode 100644
index 00000000000..3533deed578
--- /dev/null
+++ b/modules/drivers/mongo/src/metabase/driver/mongo/json.clj
@@ -0,0 +1,20 @@
+(ns metabase.driver.mongo.json
+ "This namespace adds mongo specific type encoders to `cheshire`. It is copy of the relevant part of monger's `json`
+ namespace.
+
+ TODO: I believe this namespace should be completely removed. Trying to run tests without those mongo specific
+ encoders yield no failures. Unfortunately I was unable to prove it is not needed yet, hence I'm leaving it
+ in just to be safe. Removal should be considered during follow-up of monger removal."
+ (:require
+ [cheshire.generate])
+ (:import
+ (org.bson.types BSONTimestamp ObjectId)))
+
+(set! *warn-on-reflection* true)
+
+(cheshire.generate/add-encoder ObjectId
+ (fn [^ObjectId oid ^com.fasterxml.jackson.core.json.WriterBasedJsonGenerator generator]
+ (.writeString generator (.toString oid))))
+(cheshire.generate/add-encoder BSONTimestamp
+ (fn [^BSONTimestamp ts ^com.fasterxml.jackson.core.json.WriterBasedJsonGenerator generator]
+ (cheshire.generate/encode-map {:time (.getTime ts) :inc (.getInc ts)} generator)))
diff --git a/modules/drivers/mongo/src/metabase/driver/mongo/operators.clj b/modules/drivers/mongo/src/metabase/driver/mongo/operators.clj
new file mode 100644
index 00000000000..a45ae38d43c
--- /dev/null
+++ b/modules/drivers/mongo/src/metabase/driver/mongo/operators.clj
@@ -0,0 +1,216 @@
+(ns metabase.driver.mongo.operators
+ "This namespace provides definitions of mongo operators.
+
+ Namespace is a copy of monger's operator namespace.
+
+ TODO: We should consider removing this namespace completely. Having it just adds need for maintaining list of
+ operators monger provides. We could use keywords instead. Conversion code currently handles
+ transformation of those into strings during transformations to document. More importantly -- we are already
+ using keywords in lot of places in [[metabase.driver.mongo.query-processor]]. Try seraching it for `:\\$``
+ regex.
+
+ TODO should be addressed during follow-up of monger removal."
+ {:clj-kondo/config '{:linters {:missing-docstring {:level :off}}}})
+
+(def $gt "$gt")
+(def $gte "$gte")
+(def $lt "$lt")
+(def $lte "$lte")
+(def $all "$all")
+(def $in "$in")
+(def $nin "$nin")
+(def $eq "$eq")
+(def $ne "$ne")
+(def $elemMatch "$elemMatch")
+(def $regex "$regex")
+(def $options "$options")
+(def $comment "$comment")
+(def $explain "$explain")
+(def $hint "$hint")
+(def $maxTimeMS "$maxTimeMS")
+(def $orderBy "$orderBy")
+(def $query "$query")
+(def $returnKey "$returnKey")
+(def $showDiskLoc "$showDiskLoc")
+(def $natural "$natural")
+(def $expr "$expr")
+(def $jsonSchema "$jsonSchema")
+(def $where "$where")
+(def $and "$and")
+(def $or "$or")
+(def $nor "$nor")
+(def $inc "$inc")
+(def $mul "$mul")
+(def $set "$set")
+(def $unset "$unset")
+(def $setOnInsert "$setOnInsert")
+(def $rename "$rename")
+(def $push "$push")
+(def $position "$position")
+(def $each "$each")
+(def $addToSet "$addToSet")
+(def $pop "$pop")
+(def $pull "$pull")
+(def $pullAll "$pullAll")
+(def $bit "$bit")
+(def $bitsAllClear "$bitsAllClear")
+(def $bitsAllSet "$bitsAllSet")
+(def $bitsAnyClear "$bitsAnyClear")
+(def $bitsAnySet "$bitsAnySet")
+(def $exists "$exists")
+(def $mod "$mod")
+(def $size "$size")
+(def $type "$type")
+(def $not "$not")
+(def $addFields "$addFields")
+(def $bucket "$bucket")
+(def $bucketAuto "$bucketAuto")
+(def $collStats "$collStats")
+(def $facet "$facet")
+(def $geoNear "$geoNear")
+(def $graphLookup "$graphLookup")
+(def $indexStats "$indexStats")
+(def $listSessions "$listSessions")
+(def $lookup "$lookup")
+(def $match "$match")
+(def $merge "$merge")
+(def $out "$out")
+(def $planCacheStats "$planCacheStats")
+(def $project "$project")
+(def $redact "$redact")
+(def $replaceRoot "$replaceRoot")
+(def $replaceWith "$replaceWith")
+(def $sample "$sample")
+(def $limit "$limit")
+(def $skip "$skip")
+(def $unwind "$unwind")
+(def $group "$group")
+(def $sort "$sort")
+(def $sortByCount "$sortByCount")
+(def $currentOp "$currentOp")
+(def $listLocalSessions "$listLocalSessions")
+(def $cmp "$cmp")
+(def $min "$min")
+(def $max "$max")
+(def $avg "$avg")
+(def $stdDevPop "$stdDevPop")
+(def $stdDevSamp "$stdDevSamp")
+(def $sum "$sum")
+(def $let "$let")
+(def $first "$first")
+(def $last "$last")
+(def $abs "$abs")
+(def $add "$add")
+(def $ceil "$ceil")
+(def $divide "$divide")
+(def $exp "$exp")
+(def $floor "$floor")
+(def $ln "$ln")
+(def $log "$log")
+(def $log10 "$log10")
+(def $multiply "$multiply")
+(def $pow "$pow")
+(def $round "$round")
+(def $sqrt "$sqrt")
+(def $subtract "$subtract")
+(def $trunc "$trunc")
+(def $literal "$literal")
+(def $arrayElemAt "$arrayElemAt")
+(def $arrayToObject "$arrayToObject")
+(def $concatArrays "$concatArrays")
+(def $filter "$filter")
+(def $indexOfArray "$indexOfArray")
+(def $isArray "$isArray")
+(def $map "$map")
+(def $objectToArray "$objectToArray")
+(def $range "$range")
+(def $reduce "$reduce")
+(def $reverseArray "$reverseArray")
+(def $zip "$zip")
+(def $mergeObjects "$mergeObjects")
+(def $allElementsTrue "$allElementsTrue")
+(def $anyElementsTrue "$anyElementsTrue")
+(def $setDifference "$setDifference")
+(def $setEquals "$setEquals")
+(def $setIntersection "$setIntersection")
+(def $setIsSubset "$setIsSubset")
+(def $setUnion "$setUnion")
+(def $strcasecmp "$strcasecmp")
+(def $substr "$substr")
+(def $substrBytes "$substrBytes")
+(def $substrCP "$substrCP")
+(def $toLower "$toLower")
+(def $toString "$toString")
+(def $toUpper "$toUpper")
+(def $concat "$concat")
+(def $indexOfBytes "$indexOfBytes")
+(def $indexOfCP "$indexOfCP")
+(def $ltrim "$ltrim")
+(def $regexFind "$regexFind")
+(def $regexFindAll "$regexFindAll")
+(def $regexMatch "$regexMatch")
+(def $rtrim "$rtrim")
+(def $split "$split")
+(def $strLenBytes "$strLenBytes")
+(def $subLenCP "$subLenCP")
+(def $trim "$trim")
+(def $sin "$sin")
+(def $cos "$cos")
+(def $tan "$tan")
+(def $asin "$asin")
+(def $acos "$acos")
+(def $atan "$atan")
+(def $atan2 "$atan2")
+(def $asinh "$asinh")
+(def $acosh "$acosh")
+(def $atanh "$atanh")
+(def $radiansToDegrees "$radiansToDegrees")
+(def $degreesToRadians "$degreesToRadians")
+(def $convert "$convert")
+(def $toBool "$toBool")
+(def $toDecimal "$toDecimal")
+(def $toDouble "$toDouble")
+(def $toInt "$toInt")
+(def $toLong "$toLong")
+(def $toObjectId "$toObjectId")
+(def $dayOfMonth "$dayOfMonth")
+(def $dayOfWeek "$dayOfWeek")
+(def $dayOfYear "$dayOfYear")
+(def $hour "$hour")
+(def $minute "$minute")
+(def $month "$month")
+(def $second "$second")
+(def $millisecond "$millisecond")
+(def $week "$week")
+(def $year "$year")
+(def $isoDate "$isoDate")
+(def $dateFromParts "$dateFromParts")
+(def $dateFromString "$dateFromString")
+(def $dateToParts "$dateToParts")
+(def $dateToString "$dateToString")
+(def $isoDayOfWeek "$isoDayOfWeek")
+(def $isoWeek "$isoWeek")
+(def $isoWeekYear "$isoWeekYear")
+(def $toDate "$toDate")
+(def $ifNull "$ifNull")
+(def $cond "$cond")
+(def $switch "$switch")
+(def $geoWithin "$geoWithin")
+(def $geoIntersects "$geoIntersects")
+(def $near "$near")
+(def $nearSphere "$nearSphere")
+(def $geometry "$geometry")
+(def $maxDistance "$maxDistance")
+(def $minDistance "$minDistance")
+(def $center "$center")
+(def $centerSphere "$centerSphere")
+(def $box "$box")
+(def $polygon "$polygon")
+(def $slice "$slice")
+(def $text "$text")
+(def $meta "$meta")
+(def $search "$search")
+(def $language "$language")
+(def $currentDate "$currentDate")
+(def $isolated "$isolated")
+(def $count "$count")
diff --git a/modules/drivers/mongo/src/metabase/driver/mongo/query_processor.clj b/modules/drivers/mongo/src/metabase/driver/mongo/query_processor.clj
index e218f490aa6..1f8f0c6ba22 100644
--- a/modules/drivers/mongo/src/metabase/driver/mongo/query_processor.clj
+++ b/modules/drivers/mongo/src/metabase/driver/mongo/query_processor.clj
@@ -9,6 +9,11 @@
[java-time.api :as t]
[metabase.driver :as driver]
[metabase.driver.common :as driver.common]
+ [metabase.driver.mongo.operators :refer [$add $addToSet $and $avg $concat $cond
+ $dayOfMonth $dayOfWeek $dayOfYear $divide $eq $expr
+ $group $gt $gte $hour $limit $literal $lookup $lt $lte $match $max $min
+ $minute $mod $month $multiply $ne $not $or $project $regexMatch $second
+ $size $skip $sort $strcasecmp $subtract $sum $toLower $unwind $year]]
[metabase.driver.util :as driver.u]
[metabase.lib.metadata :as lib.metadata]
[metabase.mbql.schema :as mbql.s]
@@ -25,12 +30,7 @@
[metabase.util.i18n :refer [tru]]
[metabase.util.log :as log]
[metabase.util.malli :as mu]
- [metabase.util.malli.schema :as ms]
- [monger.operators :refer [$add $addToSet $and $avg $concat $cond
- $dayOfMonth $dayOfWeek $dayOfYear $divide $eq $expr
- $group $gt $gte $hour $limit $literal $lookup $lt $lte $match $max $min $minute
- $mod $month $multiply $ne $not $or $project $regexMatch $second $size $skip $sort
- $strcasecmp $subtract $sum $toLower $unwind $year]])
+ [metabase.util.malli.schema :as ms])
(:import
(org.bson BsonBinarySubType)
(org.bson.types Binary ObjectId)))
@@ -1370,7 +1370,15 @@
"Parse a serialized native query. Like a normal JSON parse, but handles BSON/MongoDB extended JSON forms."
[^String s]
(try
- (mapv (fn [^org.bson.BsonValue v] (-> v .asDocument com.mongodb.BasicDBObject.))
+ ;; TODO: Fixme! In following expression we were previously creating BasicDBObject's. As part of Monger removal
+ ;; in favor of plain mongo-java-driver we now create Documents. I believe following conversion was and is
+ ;; responsible for https://github.com/metabase/metabase/issues/38181. When pipeline is deserialized,
+ ;; we end up with vector of `Document`s into which are appended new query stages, which are clojure
+ ;; structures. When we render the query in "view native query" in query builder, clojure structures
+ ;; are transformed to json correctly. But documents are rendered to their string representation (screenshot
+ ;; in the issue). Possible fix could be to represent native queries in ejson v2, which conforms to json rfc,
+ ;; hence there would be no need for special bson values handling. That is to be further investigated.
+ (mapv (fn [^org.bson.BsonValue v] (-> v .asDocument org.bson.Document.))
(org.bson.BsonArray/parse s))
(catch Throwable e
(throw (ex-info (tru "Unable to parse query: {0}" (.getMessage e))
diff --git a/modules/drivers/mongo/src/metabase/driver/mongo/util.clj b/modules/drivers/mongo/src/metabase/driver/mongo/util.clj
index 93a1298891e..b67a5d1948e 100644
--- a/modules/drivers/mongo/src/metabase/driver/mongo/util.clj
+++ b/modules/drivers/mongo/src/metabase/driver/mongo/util.clj
@@ -1,262 +1,96 @@
(ns metabase.driver.mongo.util
- "`*mongo-connection*`, `with-mongo-connection`, and other functions shared between several Mongo driver namespaces."
+ "Mongo specific utility functions -- mongo methods that we are using at various places wrapped into clojure
+ functions."
(:require
- [clojure.string :as str]
- [metabase.config :as config]
- [metabase.driver.util :as driver.u]
- [metabase.lib.metadata.protocols :as lib.metadata.protocols]
- [metabase.models.secret :as secret]
- [metabase.query-processor.store :as qp.store]
- [metabase.util :as u]
- [metabase.util.i18n :refer [trs tru]]
- [metabase.util.log :as log]
- [metabase.util.ssh :as ssh]
- [monger.core :as mg]
- [monger.credentials :as mcred])
+ [flatland.ordered.map :as ordered-map]
+ [metabase.driver.mongo.conversion :as mongo.conversion])
(:import
- (com.mongodb MongoClient MongoClientOptions MongoClientOptions$Builder MongoClientURI)))
+ (com.mongodb MongoClientSettings)
+ (com.mongodb.client ClientSession FindIterable MongoClient MongoClients MongoCollection MongoDatabase)))
(set! *warn-on-reflection* true)
-(def ^:dynamic ^com.mongodb.DB *mongo-connection*
- "Connection to a Mongo database. Bound by top-level `with-mongo-connection` so it may be reused within its body."
- nil)
-
-(def ^:dynamic ^com.mongodb.MongoClient *mongo-client*
- "Client used to connect to a Mongo database. Bound by top-level `with-mongo-connection` so it may be reused within its body."
- nil)
-
-;; the code below is done to support "additional connection options" the way some of the JDBC drivers do.
-;; For example, some people might want to specify a`readPreference` of `nearest`. The normal Java way of
-;; doing this would be to do
-;;
-;; (.readPreference builder (ReadPreference/nearest))
-;;
-;; But the user will enter something like `readPreference=nearest`. Luckily, the Mongo Java lib can parse
-;; these options for us and return a `MongoClientOptions` like we'd prefer. Code below:
-
-(defn- client-options-for-url-params
- "Return an instance of `MongoClientOptions` from a `url-params` string, e.g.
-
- (client-options-for-url-params \"readPreference=nearest\")
- ;; -> #MongoClientOptions{readPreference=nearest, ...}"
- ^MongoClientOptions [^String url-params]
- (when (seq url-params)
- ;; just make a fake connection string to tack the URL params on to. We can use that to have the Mongo lib
- ;; take care of parsing the params and converting them to Java-style `MongoConnectionOptions`
- (.getOptions (MongoClientURI. (str "mongodb://localhost/?" url-params)))))
-
-(defn- client-options->builder
- "Return a `MongoClientOptions.Builder` for a `MongoClientOptions` `client-options`.
- If `client-options` is `nil`, return a new 'default' builder."
- ^MongoClientOptions$Builder [^MongoClientOptions client-options]
- ;; We do it tnis way because (MongoClientOptions$Builder. nil) throws a NullPointerException
- (if client-options
- (MongoClientOptions$Builder. client-options)
- (MongoClientOptions$Builder.)))
-
-(defn- connection-options-builder
- "Build connection options for Mongo.
- We have to use `MongoClientOptions.Builder` directly to configure our Mongo connection since Monger's wrapper method
- doesn't support `.serverSelectionTimeout` or `.sslEnabled`. `additional-options`, a String like
- `readPreference=nearest`, can be specified as well; when passed, these are parsed into a `MongoClientOptions` that
- serves as a starting point for the changes made below."
- ^MongoClientOptions [{:keys [ssl additional-options ssl-cert
- ssl-use-client-auth client-ssl-cert client-ssl-key]
- :or {ssl false, ssl-use-client-auth false}}]
- (let [client-options (-> (client-options-for-url-params additional-options)
- client-options->builder
- (.description config/mb-app-id-string)
- (.connectTimeout (driver.u/db-connection-timeout-ms))
- (.serverSelectionTimeout (driver.u/db-connection-timeout-ms))
- (.sslEnabled ssl))
- server-cert? (not (str/blank? ssl-cert))
- client-cert? (and ssl-use-client-auth
- (not-any? str/blank? [client-ssl-cert client-ssl-key]))]
- (if (or server-cert? client-cert?)
- (let [ssl-params (cond-> {}
- server-cert? (assoc :trust-cert ssl-cert)
- client-cert? (assoc :private-key client-ssl-key
- :own-cert client-ssl-cert))]
- (.socketFactory client-options (driver.u/ssl-socket-factory ssl-params)))
- client-options)))
-
-;; The arglists metadata for mg/connect are actually *WRONG* -- the function additionally supports a 3-arg airity
-;; where you can pass options and credentials, as we'd like to do. We need to go in and alter the metadata of this
-;; function ourselves because otherwise the Eastwood linter will complain that we're calling the function with the
-;; wrong airity :sad: :/
-(alter-meta! #'mg/connect assoc :arglists '([{:keys [host port uri]}]
- [server-address options]
- [server-address options credentials]))
-
-(defn- database->details
- "Make sure `database` is in a standard db details format. This is done so we can accept several different types of
- values for `database`, such as plain strings or the usual MB details map."
- [database]
- (cond
- (integer? database) (qp.store/with-metadata-provider database
- (:details (lib.metadata.protocols/database (qp.store/metadata-provider))))
- (string? database) {:dbname database}
- (:dbname (:details database)) (:details database) ; entire Database obj
- (:dbname database) database ; connection details map only
- (:conn-uri database) database ; connection URI has all the parameters
- (:conn-uri (:details database)) (:details database)
- :else (throw (Exception. (str "with-mongo-connection failed: bad connection details:"
- (:details database))))))
-
-(defn- srv-conn-str
- "Creates Mongo client connection string to connect using
- DNS + SRV discovery mechanism."
- [user pass host dbname authdb]
- (format "mongodb+srv://%s:%s@%s/%s?authSource=%s" user pass host dbname authdb))
-
-(defn- normalize-details [details]
- (let [{:keys [dbname host port user pass authdb additional-options use-srv conn-uri ssl ssl-cert ssl-use-client-auth client-ssl-cert]
- :or {port 27017, ssl false, ssl-use-client-auth false, use-srv false, ssl-cert "", authdb "admin"}} details
- ;; ignore empty :user and :pass strings
- user (not-empty user)
- pass (not-empty pass)]
- {:host host
- :port port
- :user user
- :authdb authdb
- :pass pass
- :dbname dbname
- :ssl ssl
- :additional-options additional-options
- :conn-uri conn-uri
- :srv? use-srv
- :ssl-cert ssl-cert
- :ssl-use-client-auth ssl-use-client-auth
- :client-ssl-cert client-ssl-cert
- :client-ssl-key (secret/get-secret-string details "client-ssl-key")}))
-
-(defn- fqdn?
- "A very simple way to check if a hostname is fully-qualified:
- Check if there are two or more periods in the name."
- [host]
- (<= 2 (-> host frequencies (get \. 0))))
-
-(defn- auth-db-or-default
- "Returns the auth-db to use for a connection, for the given `auth-db` parameter. If `auth-db` is a non-blank string,
- it will be returned. Otherwise, the default value (\"admin\") will be returned."
- [auth-db]
- (if (str/blank? auth-db) "admin" auth-db))
-
-(defn- srv-connection-info
- "Connection info for Mongo using DNS SRV. Requires FQDN for `host` in the format
- 'subdomain. ... .domain.top-level-domain'. Only a single host is supported, but a
- replica list could easily provided instead of a single host.
- Using SRV automatically enables SSL, though we explicitly set SSL to true anyway.
- Docs to generate URI string: https://docs.mongodb.com/manual/reference/connection-string/#dns-seedlist-connection-format"
- [{:keys [host user authdb pass dbname] :as details}]
- (if-not (fqdn? host)
- (throw (ex-info (tru "Using DNS SRV requires a FQDN for host")
- {:host host}))
- (let [conn-opts (connection-options-builder details)
- authdb (auth-db-or-default authdb)
- conn-str (srv-conn-str user pass host dbname authdb)]
- {:type :srv
- :uri (MongoClientURI. conn-str conn-opts)})))
-
-(defn- normal-connection-info
- "Connection info for Mongo. Returns options for the fallback method to connect
- to hostnames that are not FQDNs. This works with 'localhost', but has been problematic with FQDNs.
- If you would like to provide a FQDN, use `srv-connection-info`"
- [{:keys [host port user authdb pass dbname] :as details}]
- (let [server-address (mg/server-address host port)
- credentials (when user
- (mcred/create user (auth-db-or-default authdb) pass))
- ^MongoClientOptions$Builder opts (connection-options-builder details)]
- {:type :normal
- :server-address server-address
- :credentials credentials
- :dbname dbname
- :options (-> opts .build)}))
-
-(defn- conn-string-info
- "Connection info for Mongo using a user-provided connection string."
- [{:keys [conn-uri]}]
- {:type :conn-string
- :conn-string conn-uri})
-
-(defn- details->mongo-connection-info [{:keys [conn-uri srv?], :as details}]
- (if (str/blank? conn-uri)
- ((if srv?
- srv-connection-info
- normal-connection-info) details)
- (conn-string-info details)))
-
-(defmulti ^:private connect
- "Connect to MongoDB using Mongo `connection-info`, return a tuple of `[mongo-client db]`, instances of `MongoClient`
- and `DB` respectively.
-
- If `host` is a fully-qualified domain name, then we need to connect to Mongo
- differently. It has been problematic to connect to Mongo with an FQDN using
- `mg/connect`. The fix was to create a connection string and use DNS SRV for
- FQDNS. In this fn we provide the correct connection fn based on host."
- {:arglists '([connection-info])}
- :type)
-
-(defmethod connect :srv
- [{:keys [^MongoClientURI uri]}]
- (let [mongo-client (MongoClient. uri)]
- (if-let [db-name (.getDatabase uri)]
- [mongo-client (.getDB mongo-client db-name)]
- (throw (ex-info (tru "No database name specified in URI. Monger requires a database to be explicitly configured.")
- {:hosts (-> uri .getHosts)
- :uri (-> uri .getURI)
- :opts (-> uri .getOptions)})))))
-
-(defmethod connect :normal
- [{:keys [server-address options credentials dbname]}]
- (let [do-connect (partial mg/connect server-address options)
- mongo-client (if credentials
- (do-connect credentials)
- (do-connect))]
- [mongo-client (mg/get-db mongo-client dbname)]))
-
-(defmethod connect :conn-string
- [{:keys [conn-string]}]
- (let [mongo-client (mg/connect-via-uri conn-string)]
- [(:conn mongo-client) (:db mongo-client)]))
-
-(defn do-with-mongo-connection
- "Run `f` with a new connection (bound to [[*mongo-connection*]]) to `database`. Don't use this directly; use
- [[with-mongo-connection]]. Also dynamically binds the Mongo client to [[*mongo-client*]]."
- [f database]
- (let [details (database->details database)]
- (ssh/with-ssh-tunnel [details-with-tunnel details]
- (let [connection-info (details->mongo-connection-info (normalize-details details-with-tunnel))
- [mongo-client db] (connect connection-info)]
- (log/debug (u/format-color 'cyan (trs "Opened new MongoDB connection.")))
- (try
- (binding [*mongo-connection* db
- *mongo-client* mongo-client]
- (f *mongo-connection*))
- (finally
- (mg/disconnect mongo-client)
- (log/debug (u/format-color 'cyan (trs "Closed MongoDB connection.")))))))))
-
-(defmacro with-mongo-connection
- "Open a new MongoDB connection to ``database-or-connection-string`, bind connection to `binding`, execute `body`, and
- close the connection. The DB connection is re-used by subsequent calls to [[with-mongo-connection]] within
- `body`. (We're smart about it: `database` isn't even evaluated if [[*mongo-connection*]] is already bound.)
-
- [[*mongo-client*]] is also dynamically bound to the MongoClient instance.
-
- ;; delay isn't derefed if *mongo-connection* is already bound
- (with-mongo-connection [^com.mongodb.DB conn @(:db (sel :one Table ...))]
- ...)
-
- ;; You can use a string instead of a Database
- (with-mongo-connection [^com.mongodb.DB conn \"mongodb://127.0.0.1:27017/test\"]
- ...)
-
- `database-or-connection-string` can also optionally be the connection details map on its own."
- [[database-binding database] & body]
- `(let [f# (fn [~database-binding]
- ~@body)]
- (if *mongo-connection*
- (f# *mongo-connection*)
- (do-with-mongo-connection f# ~database))))
+(defn mongo-client
+ "Create `MongoClient` from `MongoClientSettings`."
+ ^MongoClient [^MongoClientSettings settings]
+ (MongoClients/create settings))
+
+(defn close
+ "Close `client`."
+ [^MongoClient client]
+ (.close client))
+
+(defn database
+ "Get database by its name from `client`."
+ ^MongoDatabase [^MongoClient client db-name]
+ (.getDatabase client db-name))
+
+;; https://mongodb.github.io/mongo-java-driver/4.11/apidocs/mongodb-driver-sync/com/mongodb/client/MongoDatabase.html#runCommand(org.bson.conversions.Bson)
+;; Returns Document
+(defn run-command
+ "Run mongo command."
+ ([^MongoDatabase db cmd & {:as opts}]
+ (let [cmd-doc (mongo.conversion/to-document cmd)]
+ (-> (.runCommand db cmd-doc)
+ (mongo.conversion/from-document opts)))))
+
+;; https://mongodb.github.io/mongo-java-driver/4.11/apidocs/mongodb-driver-sync/com/mongodb/client/MongoClient.html#listDatabaseNames()
+;; returns MongoIterable<String>
+(defn list-database-names
+ "Return vector of names of databases for `client`."
+ [^MongoClient client]
+ (vec (.listDatabaseNames client)))
+
+;; https://mongodb.github.io/mongo-java-driver/4.11/apidocs/mongodb-driver-sync/com/mongodb/client/MongoDatabase.html#listCollectionNames()
+(defn list-collection-names
+ "Return vector of collection names for `db`"
+ [^MongoDatabase db]
+ (vec (.listCollectionNames db)))
+
+(defn collection
+ "Return `MongoCollection` for `db` by its name."
+ ^MongoCollection [^MongoDatabase db coll-name]
+ (.getCollection db coll-name))
+
+;; https://mongodb.github.io/mongo-java-driver/4.11/apidocs/mongodb-driver-sync/com/mongodb/client/MongoCollection.html#listIndexes()
+(defn list-indexes
+ "Return vector of Documets describing indexes."
+ [^MongoCollection coll & {:as opts}]
+ (mongo.conversion/from-document (.listIndexes coll) opts))
+
+;; https://mongodb.github.io/mongo-java-driver/4.11/apidocs/mongodb-driver-sync/com/mongodb/client/MongoCollection.html#find()
+(defn do-find
+ "Perform find on collection. `sort-criteria` should be sequence of key value pairs (eg. vector of vectors), or
+ `ordered-map`. Keys are the column name. Keys could be keywords. `opts` could contain also `:keywordize`, which
+ is param for `from-document`."
+ [^MongoCollection coll
+ & {:keys [limit skip batch-size sort-criteria] :as opts}]
+ (->> (cond-> ^FindIterable (.find coll)
+ limit (.limit limit)
+ skip (.skip skip)
+ batch-size (.batchSize (int batch-size))
+ sort-criteria (.sort (mongo.conversion/to-document (ordered-map/ordered-map sort-criteria))))
+ (mapv #(mongo.conversion/from-document % opts))))
+
+(defn create-index
+ "Create index."
+ [^MongoCollection coll cmd-map]
+ (.createIndex coll (mongo.conversion/to-document cmd-map)))
+
+(defn insert-one
+ "Insert document into mongo collection."
+ [^MongoCollection coll document-map]
+ (.insertOne coll (mongo.conversion/to-document document-map)))
+
+(defn start-session!
+ "Start session on client `c`."
+ ^ClientSession [^MongoClient c]
+ (. c startSession))
+
+(defn kill-session!
+ "Kill `session` in `db`."
+ [^MongoDatabase db
+ ^ClientSession session]
+ (let [session-id (.. session getServerSession getIdentifier)
+ kill-cmd (mongo.conversion/to-document {:killSessions [session-id]})]
+ (.runCommand db kill-cmd)))
diff --git a/modules/drivers/mongo/test/metabase/driver/mongo/connection_test.clj b/modules/drivers/mongo/test/metabase/driver/mongo/connection_test.clj
new file mode 100644
index 00000000000..2030d27f3a8
--- /dev/null
+++ b/modules/drivers/mongo/test/metabase/driver/mongo/connection_test.clj
@@ -0,0 +1,141 @@
+(ns metabase.driver.mongo.connection-test
+ (:require
+ [clojure.string :as str]
+ [clojure.test :refer :all]
+ [metabase.driver.mongo.connection :as mongo.connection]
+ [metabase.driver.mongo.database :as mongo.db]
+ [metabase.driver.mongo.util :as mongo.util]
+ [metabase.driver.util :as driver.u]
+ [metabase.test :as mt])
+ (:import
+ (com.mongodb ServerAddress)
+ (com.mongodb.client MongoDatabase)))
+
+;;;; TODO: move some tests to db-test?
+
+(set! *warn-on-reflection* true)
+
+(def ^:private mock-details
+ {:user "test-user"
+ :pass "test-pass"
+ :host "test-host.place.com"
+ :dbname "datadb"
+ :authdb "authdb"
+ :use-srv true})
+
+(deftest ^:parallel fqdn?-test
+ (testing "test hostname is fqdn"
+ (is (true? (#'mongo.db/fqdn? "db.mongo.com")))
+ (is (true? (#'mongo.db/fqdn? "replica-01.db.mongo.com")))
+ (is (false? (#'mongo.db/fqdn? "localhost")))
+ (is (false? (#'mongo.db/fqdn? "localhost.localdomain")))))
+
+(deftest ^:parallel srv-conn-str-test
+ (let [db-details {:user "test-user"
+ :pass "test-pass"
+ :host "test-host.place.com"
+ :dbname "datadb"
+ :authdb "authdb"
+ :use-srv true}]
+ (testing "mongo+srv connection string used when :use-srv is thruthy"
+ (is (str/includes? (mongo.connection/db-details->connection-string db-details)
+ "mongodb+srv://test-user:test-pass@test-host.place.com/datadb?authSource=authdb")))
+ (testing "Only fqdn may be used with mongo+srv"
+ (is (thrown-with-msg? Throwable
+ #"Using DNS SRV requires a FQDN for host"
+ (-> db-details
+ (assoc :host "localhost")
+ mongo.db/details-normalized
+ mongo.connection/db-details->connection-string))))))
+
+(deftest ^:parallel srv-connection-properties-test
+ (testing "connection properties when using SRV"
+ (are [host msg]
+ (thrown-with-msg? Throwable msg
+ (mongo.connection/with-mongo-database [^MongoDatabase db
+ {:host host
+ :port 1015
+ :user "test-user"
+ :authdb "test-authdb"
+ :pass "test-passwd"
+ :dbname "test-dbname"
+ :ssl true
+ :additional-options "connectTimeoutMS=2000&serverSelectionTimeoutMS=2000"
+ :use-srv true}]
+ (mongo.util/list-collection-names db)))
+ "db.fqdn.test" #"Failed looking up SRV record"
+ "local.test" #"Using DNS SRV requires a FQDN for host")
+ (testing "test host and port are correct for both srv and normal"
+ (let [host "localhost"
+ details {:host host
+ :port 1010
+ :user "test-user"
+ :authdb "test-authdb"
+ :pass "test-passwd"
+ :dbname "test-dbname"
+ :ssl true
+ :additional-options ""}
+ client-settings (mongo.connection/db-details->mongo-client-settings details)
+ ^ServerAddress server-address (-> client-settings .getClusterSettings .getHosts first)]
+ (is (= "localhost"
+ (.getHost server-address)))
+ (is (= 1010
+ (.getPort server-address)))))))
+
+(deftest ^:parallel additional-connection-options-test
+ (mt/test-driver
+ :mongo
+ (testing "test that people can specify additional connection options like `?readPreference=nearest`"
+ (is (= (com.mongodb.ReadPreference/nearest)
+ (-> (assoc mock-details :additional-options "readPreference=nearest")
+ mongo.connection/db-details->mongo-client-settings
+ .getReadPreference)))
+ (is (= (com.mongodb.ReadPreference/secondaryPreferred)
+ (-> mock-details
+ (assoc :additional-options "readPreference=secondaryPreferred")
+ mongo.connection/db-details->mongo-client-settings
+ .getReadPreference))))
+ (testing "make sure we can specify multiple options"
+ (let [settings (-> mock-details
+ (assoc :additional-options "readPreference=secondary&replicaSet=test")
+ mongo.connection/db-details->mongo-client-settings)]
+ (is (= "test"
+ (-> settings .getClusterSettings .getRequiredReplicaSetName)))
+ (is (= (com.mongodb.ReadPreference/secondary)
+ (.getReadPreference settings)))))
+ (testing "make sure that invalid additional options throw an Exception"
+ (is (thrown-with-msg?
+ IllegalArgumentException
+ #"No match for read preference of ternary"
+ (-> mock-details
+ (assoc :additional-options "readPreference=ternary")
+ mongo.connection/db-details->mongo-client-settings))))))
+
+(deftest ^:parallel test-ssh-connection
+ (testing "Gets an error when it can't connect to mongo via ssh tunnel"
+ (mt/test-driver
+ :mongo
+ (is (thrown?
+ java.net.ConnectException
+ (try
+ (let [engine :mongo
+ details {:ssl false
+ :password "changeme"
+ :tunnel-host "localhost"
+ :tunnel-pass "BOGUS-BOGUS"
+ :port 5432
+ :dbname "test"
+ :host "localhost"
+ :tunnel-enabled true
+ ;; we want to use a bogus port here on purpose -
+ ;; so that locally, it gets a ConnectionRefused,
+ ;; and in CI it does too. Apache's SSHD library
+ ;; doesn't wrap every exception in an SshdException
+ :tunnel-port 21212
+ :tunnel-user "bogus"}]
+ (driver.u/can-connect-with-details? engine details :throw-exceptions))
+ (catch Throwable e
+ (loop [^Throwable e e]
+ (or (when (instance? java.net.ConnectException e)
+ (throw e))
+ (some-> (.getCause e) recur))))))))))
diff --git a/modules/drivers/mongo/test/metabase/driver/mongo/conversion_test.clj b/modules/drivers/mongo/test/metabase/driver/mongo/conversion_test.clj
new file mode 100644
index 00000000000..9ead5932b56
--- /dev/null
+++ b/modules/drivers/mongo/test/metabase/driver/mongo/conversion_test.clj
@@ -0,0 +1,28 @@
+(ns metabase.driver.mongo.conversion-test
+ (:require
+ [clojure.test :refer :all]
+ [clojure.walk :as walk]
+ [flatland.ordered.map :as ordered-map]
+ [metabase.driver.mongo.conversion :as mongo.conversion]))
+
+(set! *warn-on-reflection* true)
+
+(deftest ^:parallel transformation-test
+ (let [m (ordered-map/ordered-map
+ :a 1
+ :b "x"
+ :c {:d "e"})
+ ms {"u" "v"
+ "x" {"y" "z"}}]
+ (testing "Transform map"
+ (is (= m
+ (-> m mongo.conversion/to-document (mongo.conversion/from-document {:keywordize true}))))
+ (is (= ms
+ (-> ms mongo.conversion/to-document (mongo.conversion/from-document nil)))))
+ (testing "Transform sequence"
+ (let [mseqk [m (walk/keywordize-keys ms)]
+ mseqs [(walk/stringify-keys m) ms]]
+ (is (= mseqk
+ (-> mseqk mongo.conversion/to-document (mongo.conversion/from-document {:keywordize true}))))
+ (is (= mseqs
+ (-> mseqs mongo.conversion/to-document (mongo.conversion/from-document nil))))))))
diff --git a/modules/drivers/mongo/test/metabase/driver/mongo/database_test.clj b/modules/drivers/mongo/test/metabase/driver/mongo/database_test.clj
new file mode 100644
index 00000000000..1b3a2cfb3e7
--- /dev/null
+++ b/modules/drivers/mongo/test/metabase/driver/mongo/database_test.clj
@@ -0,0 +1,19 @@
+(ns metabase.driver.mongo.database-test
+ (:require
+ [clojure.test :refer :all]
+ [metabase.driver.mongo.database :as mongo.db]))
+
+(deftest ^:parallel fqdn?-test
+ (testing "test hostname is fqdn"
+ (is (true? (#'mongo.db/fqdn? "db.mongo.com")))
+ (is (true? (#'mongo.db/fqdn? "replica-01.db.mongo.com")))
+ (is (false? (#'mongo.db/fqdn? "localhost")))
+ (is (false? (#'mongo.db/fqdn? "localhost.localdomain")))))
+
+(deftest ^:parallel db-name-test
+ (testing "`dbname` is in db-details"
+ (is (= "some_db"
+ (mongo.db/db-name {:dbname "some_db"}))))
+ (testing "`dbname` is encoded in conn-uri"
+ (is (= "some_db"
+ (mongo.db/db-name {:conn-uri "mongodb://localhost/some_db"})))))
diff --git a/modules/drivers/mongo/test/metabase/driver/mongo/execute_test.clj b/modules/drivers/mongo/test/metabase/driver/mongo/execute_test.clj
index aa2d52e7a85..a4dd06b6362 100644
--- a/modules/drivers/mongo/test/metabase/driver/mongo/execute_test.clj
+++ b/modules/drivers/mongo/test/metabase/driver/mongo/execute_test.clj
@@ -3,12 +3,13 @@
[clojure.core.async :as a]
[clojure.test :refer :all]
[metabase.async.streaming-response :as streaming-response]
+ [metabase.driver.mongo.conversion :as mongo.conversion]
[metabase.driver.mongo.execute :as mongo.execute]
[metabase.query-processor :as qp]
[metabase.query-processor.context :as qp.context]
[metabase.test :as mt])
(:import
- (com.mongodb BasicDBObject)
+ #_(com.mongodb BasicDBObject)
(java.util NoSuchElementException)))
(set! *warn-on-reflection* true)
@@ -20,7 +21,7 @@
(next [_] (let [i @counter]
(vswap! counter inc)
(if (< i (count rows))
- (BasicDBObject. ^java.util.Map (get rows i))
+ (mongo.conversion/to-document (get rows i))
(throw (NoSuchElementException. (str "no element at " i))))))
(close [_]))))
diff --git a/modules/drivers/mongo/test/metabase/driver/mongo/util_test.clj b/modules/drivers/mongo/test/metabase/driver/mongo/util_test.clj
deleted file mode 100644
index 49b3e7678ad..00000000000
--- a/modules/drivers/mongo/test/metabase/driver/mongo/util_test.clj
+++ /dev/null
@@ -1,172 +0,0 @@
-(ns metabase.driver.mongo.util-test
- (:require
- [clojure.test :refer :all]
- [metabase.driver.mongo.util :as mongo.util]
- [metabase.driver.util :as driver.u]
- [metabase.test :as mt])
- (:import
- (com.mongodb MongoClient MongoClientOptions$Builder ReadPreference ServerAddress)))
-
-(set! *warn-on-reflection* true)
-
-(defn- connect-mongo ^MongoClient [opts]
- (let [connection-info (#'mongo.util/details->mongo-connection-info
- (#'mongo.util/normalize-details
- opts))]
- (#'mongo.util/connect connection-info)))
-
-(def connect-passthrough
- (fn [{map-type :type}]
- map-type))
-
-(def srv-passthrough
- (fn [_] {:type :srv}))
-
-(deftest ^:parallel fqdn?-test
- (testing "test hostname is fqdn"
- (is (true? (#'mongo.util/fqdn? "db.mongo.com")))
- (is (true? (#'mongo.util/fqdn? "replica-01.db.mongo.com")))
- (is (false? (#'mongo.util/fqdn? "localhost")))
- (is (false? (#'mongo.util/fqdn? "localhost.localdomain")))))
-
-(deftest ^:parallel srv-conn-str-test
- (testing "test srv connection string"
- (is (= "mongodb+srv://test-user:test-pass@test-host.place.com/datadb?authSource=authdb"
- (#'mongo.util/srv-conn-str "test-user" "test-pass" "test-host.place.com" "datadb" "authdb")))))
-
-(deftest srv-toggle-test
- (testing "test that srv toggle works"
- (is (= :srv
- (with-redefs [mongo.util/srv-connection-info srv-passthrough
- mongo.util/connect connect-passthrough]
- (let [host "my.fake.domain"
- opts {:host host
- :port 1015
- :user "test-user"
- :authdb "test-authdb"
- :pass "test-passwd"
- :dbname "test-dbname"
- :ssl true
- :additional-options ""
- :use-srv true}]
- (connect-mongo opts)))))
-
- (is (= :normal
- (with-redefs [mongo.util/connect connect-passthrough]
- (let [host "localhost"
- opts {:host host
- :port 1010
- :user "test-user"
- :authdb "test-authdb"
- :pass "test-passwd"
- :dbname "test-dbname"
- :ssl true
- :additional-options ""
- :use-srv false}]
- (connect-mongo opts)))))
-
- (is (= :normal
- (with-redefs [mongo.util/connect connect-passthrough]
- (let [host "localhost.domain"
- opts {:host host
- :port 1010
- :user "test-user"
- :authdb "test-authdb"
- :pass "test-passwd"
- :dbname "test-dbname"
- :ssl true
- :additional-options ""}]
- (connect-mongo opts)))))))
-
-(deftest ^:parallel srv-connection-properties-test
- (testing "connection properties when using SRV"
- (are [host msg] (thrown-with-msg? Throwable msg
- (connect-mongo {:host host
- :port 1015
- :user "test-user"
- :authdb "test-authdb"
- :pass "test-passwd"
- :dbname "test-dbname"
- :ssl true
- :additional-options ""
- :use-srv true}))
- "db.fqdn.test" #"Unable to look up TXT record for host db.fqdn.test"
- "local.test" #"Using DNS SRV requires a FQDN for host")
-
- (testing "test host and port are correct for both srv and normal"
- (let [host "localhost"
- opts {:host host
- :port 1010
- :user "test-user"
- :authdb "test-authdb"
- :pass "test-passwd"
- :dbname "test-dbname"
- :ssl true
- :additional-options ""}
- [^MongoClient mongo-client _db] (connect-mongo opts)
- ^ServerAddress mongo-addr (-> mongo-client
- (.getAllAddress)
- first)
- mongo-host (-> mongo-addr .getHost)
- mongo-port (-> mongo-addr .getPort)]
- (is (= "localhost"
- mongo-host))
- (is (= 1010
- mongo-port))))))
-
-(defn- connection-options-builder ^MongoClientOptions$Builder [details]
- (#'mongo.util/connection-options-builder details))
-
-(deftest ^:parallel additional-connection-options-test
- (testing "test that people can specify additional connection options like `?readPreference=nearest`"
- (is (= (ReadPreference/nearest)
- (.getReadPreference (-> (connection-options-builder {:additional-options "readPreference=nearest"})
- .build))))
-
- (is (= (ReadPreference/secondaryPreferred)
- (.getReadPreference (-> (connection-options-builder {:additional-options "readPreference=secondaryPreferred"})
- .build))))
-
- (testing "make sure we can specify multiple options"
- (let [opts (-> (connection-options-builder {:additional-options "readPreference=secondary&replicaSet=test"})
- .build)]
- (is (= "test"
- (.getRequiredReplicaSetName opts)))
-
- (is (= (ReadPreference/secondary)
- (.getReadPreference opts)))))
-
- (testing "make sure that invalid additional options throw an Exception"
- (is (thrown-with-msg?
- IllegalArgumentException
- #"No match for read preference of ternary"
- (-> (connection-options-builder {:additional-options "readPreference=ternary"})
- .build))))))
-
-(deftest ^:parallel test-ssh-connection
- (testing "Gets an error when it can't connect to mongo via ssh tunnel"
- (mt/test-driver :mongo
- (is (thrown?
- java.net.ConnectException
- (try
- (let [engine :mongo
- details {:ssl false
- :password "changeme"
- :tunnel-host "localhost"
- :tunnel-pass "BOGUS-BOGUS"
- :port 5432
- :dbname "test"
- :host "localhost"
- :tunnel-enabled true
- ;; we want to use a bogus port here on purpose -
- ;; so that locally, it gets a ConnectionRefused,
- ;; and in CI it does too. Apache's SSHD library
- ;; doesn't wrap every exception in an SshdException
- :tunnel-port 21212
- :tunnel-user "bogus"}]
- (driver.u/can-connect-with-details? engine details :throw-exceptions))
- (catch Throwable e
- (loop [^Throwable e e]
- (or (when (instance? java.net.ConnectException e)
- (throw e))
- (some-> (.getCause e) recur))))))))))
diff --git a/modules/drivers/mongo/test/metabase/driver/mongo_test.clj b/modules/drivers/mongo/test/metabase/driver/mongo_test.clj
index f50e697ab0c..d644abfda68 100644
--- a/modules/drivers/mongo/test/metabase/driver/mongo_test.clj
+++ b/modules/drivers/mongo/test/metabase/driver/mongo_test.clj
@@ -8,6 +8,7 @@
[metabase.db.metadata-queries :as metadata-queries]
[metabase.driver :as driver]
[metabase.driver.mongo :as mongo]
+ [metabase.driver.mongo.connection :as mongo.connection]
[metabase.driver.mongo.query-processor :as mongo.qp]
[metabase.driver.mongo.util :as mongo.util]
[metabase.driver.util :as driver.u]
@@ -25,8 +26,6 @@
[metabase.test.data.interface :as tx]
[metabase.test.data.mongo :as tdm]
[metabase.util.log :as log]
- [monger.collection :as mcoll]
- [monger.core :as mg]
[taoensso.nippy :as nippy]
[toucan2.core :as t2]
[toucan2.tools.with-temp :as t2.with-temp])
@@ -73,9 +72,11 @@
:port 3000
:dbname "bad-db-name?connectTimeoutMS=50"}
:expected false}
- {:details {:conn-uri "mongodb://metabase:metasample123@localhost:27017/test-data?authSource=admin"}
+ {:details {:use-conn-uri true
+ :conn-uri "mongodb://metabase:metasample123@localhost:27017/test-data?authSource=admin"}
:expected (not (tdm/ssl-required?))}
- {:details {:conn-uri "mongodb://localhost:3000/bad-db-name?connectTimeoutMS=50"}
+ {:details {:use-conn-uri true
+ :conn-uri "mongodb://localhost:3000/bad-db-name?connectTimeoutMS=50"}
:expected false}]
:let [ssl-details (tdm/conn-details details)]]
(testing (str "connect with " details)
@@ -244,16 +245,16 @@
(is (true? (t2/select-one-fn :database_indexed :model/Field (mt/id :singly-index :indexed))))
(is (false? (t2/select-one-fn :database_indexed :model/Field (mt/id :singly-index :not-indexed)))))
- (testing "compount index"
- (mongo.util/with-mongo-connection [conn (mt/db)]
- (mcoll/create-index conn "compound-index" (array-map "first" 1 "second" 1)))
- (sync/sync-database! (mt/db))
- (is (true? (t2/select-one-fn :database_indexed :model/Field (mt/id :compound-index :first))))
- (is (false? (t2/select-one-fn :database_indexed :model/Field (mt/id :compound-index :second)))))
+ (testing "compount index"
+ (mongo.connection/with-mongo-database [db (mt/db)]
+ (mongo.util/create-index (mongo.util/collection db "compound-index") (array-map "first" 1 "second" 1)))
+ (sync/sync-database! (mt/db))
+ (is (true? (t2/select-one-fn :database_indexed :model/Field (mt/id :compound-index :first))))
+ (is (false? (t2/select-one-fn :database_indexed :model/Field (mt/id :compound-index :second)))))
(testing "multi key index"
- (mongo.util/with-mongo-connection [conn (mt/db)]
- (mcoll/create-index conn "multi-key-index" (array-map "url.small" 1)))
+ (mongo.connection/with-mongo-database [db (mt/db)]
+ (mongo.util/create-index (mongo.util/collection db "multi-key-index") (array-map "url.small" 1)))
(sync/sync-database! (mt/db))
(is (false? (t2/select-one-fn :database_indexed :model/Field :name "url")))
(is (true? (t2/select-one-fn :database_indexed :model/Field :name "small"))))
@@ -286,38 +287,41 @@
(try
(let [describe-indexes (fn [table-name]
(driver/describe-table-indexes :mongo (mt/db) (t2/select-one :model/Table (mt/id table-name))))]
- (mongo.util/with-mongo-connection [conn (mt/db)]
+ (mongo.connection/with-mongo-database [db (mt/db)]
(testing "single column index"
- (mcoll/create-index conn "singly-index" {"a" 1})
+ (mongo.util/create-index (mongo.util/collection db "singly-index") {"a" 1})
(is (= #{{:type :normal-column-index :value "_id"}
{:type :normal-column-index :value "a"}}
(describe-indexes :singly-index))))
(testing "compound index column index"
- (mcoll/create-index conn "compound-index" (array-map "a" 1 "b" 1 "c" 1)) ;; first index column is :a
- (mcoll/create-index conn "compound-index" (array-map "e" 1 "d" 1 "f" 1)) ;; first index column is :e
+ ;; first index column is :a
+ (mongo.util/create-index (mongo.util/collection db "compound-index") (array-map :a 1 :b 1 :c 1))
+ ;; first index column is :e
+ (mongo.util/create-index (mongo.util/collection db "compound-index") (array-map :e 1 :d 1 :f 1))
(is (= #{{:type :normal-column-index :value "_id"}
{:type :normal-column-index :value "a"}
{:type :normal-column-index :value "e"}}
(describe-indexes :compound-index))))
(testing "compound index that has many keys can still determine the first key"
- (mcoll/create-index conn "compound-index-big"
- (array-map "j" 1 "b" 1 "c" 1 "d" 1 "e" 1 "f" 1 "g" 1 "h" 1 "a" 1)) ;; first index column is :j
+ ;; first index column is :j
+ (mongo.util/create-index (mongo.util/collection db "compound-index-big")
+ (array-map "j" 1 "b" 1 "c" 1 "d" 1 "e" 1 "f" 1 "g" 1 "h" 1 "a" 1))
(is (= #{{:type :normal-column-index :value "_id"}
{:type :normal-column-index :value "j"}}
(describe-indexes :compound-index-big))))
(testing "multi key indexes"
- (mcoll/create-index conn "multi-key-index" (array-map "a.b" 1))
+ (mongo.util/create-index (mongo.util/collection db "multi-key-index") (array-map "a.b" 1))
(is (= #{{:type :nested-column-index :value ["a" "b"]}
{:type :normal-column-index :value "_id"}}
(describe-indexes :multi-key-index))))
(testing "advanced-index: hashed index, text index, geospatial index"
- (mcoll/create-index conn "advanced-index" (array-map "hashed-field" "hashed"))
- (mcoll/create-index conn "advanced-index" (array-map "text-field" "text"))
- (mcoll/create-index conn "advanced-index" (array-map "geospatial-field" "2d"))
+ (mongo.util/create-index (mongo.util/collection db "advanced-index") (array-map "hashed-field" "hashed"))
+ (mongo.util/create-index (mongo.util/collection db "advanced-index") (array-map "text-field" "text"))
+ (mongo.util/create-index (mongo.util/collection db "advanced-index") (array-map "geospatial-field" "2d"))
(is (= #{{:type :normal-column-index :value "geospatial-field"}
{:type :normal-column-index :value "hashed-field"}
{:type :normal-column-index :value "_id"}
@@ -603,17 +607,18 @@
(tx/destroy-db! :mongo dbdef)
(let [details (tx/dbdef->connection-details :mongo :db dbdef)]
;; load rows
- (mongo.util/with-mongo-connection [conn details]
- (doseq [[i row] (map-indexed vector row-maps)
- :let [row (assoc row :_id (inc i))]]
- (try
- (mcoll/insert conn collection-name row)
- (catch Throwable e
- (throw (ex-info (format "Error inserting row: %s" (ex-message e))
- {:database database-name, :collection collection-name, :details details, :row row}
- e)))))
- (log/infof "Inserted %d rows into %s collection %s."
- (count row-maps) (pr-str database-name) (pr-str collection-name)))
+ (mongo.connection/with-mongo-database [db details]
+ (let [coll (mongo.util/collection db collection-name)]
+ (doseq [[i row] (map-indexed vector row-maps)
+ :let [row (assoc row :_id (inc i))]]
+ (try
+ (mongo.util/insert-one coll row)
+ (catch Throwable e
+ (throw (ex-info (format "Error inserting row: %s" (ex-message e))
+ {:database database-name, :collection collection-name, :details details, :row row}
+ e)))))
+ (log/infof "Inserted %d rows into %s collection %s."
+ (count row-maps) (pr-str database-name) (pr-str collection-name))))
;; now sync the Database.
(let [db (first (t2/insert-returning-instances! Database {:name database-name, :engine "mongo", :details details}))]
(sync/sync-database! db)
@@ -662,15 +667,15 @@
(deftest strange-versionArray-test
(mt/test-driver :mongo
(testing "Negative values in versionArray are ignored (#29678)"
- (with-redefs [mg/command (constantly {"version" "4.0.28-23"
- "versionArray" [4 0 29 -100]})]
+ (with-redefs [mongo.util/run-command (constantly {"version" "4.0.28-23"
+ "versionArray" [4 0 29 -100]})]
(is (= {:version "4.0.28-23"
:semantic-version [4 0 29]}
(driver/dbms-version :mongo (mt/db))))))
(testing "Any values after rubbish in versionArray are ignored"
- (with-redefs [mg/command (constantly {"version" "4.0.28-23"
- "versionArray" [4 0 "NaN" 29]})]
+ (with-redefs [mongo.util/run-command (constantly {"version" "4.0.28-23"
+ "versionArray" [4 0 "NaN" 29]})]
(is (= {:version "4.0.28-23"
:semantic-version [4 0]}
(driver/dbms-version :mongo (mt/db))))))))
diff --git a/modules/drivers/mongo/test/metabase/test/data/mongo.clj b/modules/drivers/mongo/test/metabase/test/data/mongo.clj
index 33a4bfacec0..b09c1858766 100644
--- a/modules/drivers/mongo/test/metabase/test/data/mongo.clj
+++ b/modules/drivers/mongo/test/metabase/test/data/mongo.clj
@@ -9,12 +9,12 @@
[metabase.config :as config]
[metabase.driver :as driver]
[metabase.driver.ddl.interface :as ddl.i]
- [metabase.driver.mongo.util :refer [with-mongo-connection]]
- [metabase.test.data.interface :as tx]
- [monger.collection :as mcoll]
- [monger.core :as mg])
+ [metabase.driver.mongo.connection :as mongo.connection]
+ [metabase.driver.mongo.util :as mongo.util]
+ [metabase.test.data.interface :as tx])
(:import
- (com.fasterxml.jackson.core JsonGenerator)))
+ (com.fasterxml.jackson.core JsonGenerator)
+ (com.mongodb.client MongoDatabase)))
(set! *warn-on-reflection* true)
@@ -58,8 +58,8 @@
{:pass password}))))
(defn- destroy-db! [driver dbdef]
- (with-mongo-connection [^com.mongodb.DB mongo-connection (tx/dbdef->connection-details driver :server dbdef)]
- (mg/drop-db (.getMongo mongo-connection) (tx/escaped-database-name dbdef))))
+ (mongo.connection/with-mongo-database [^MongoDatabase db (tx/dbdef->connection-details driver :server dbdef)]
+ (.drop db)))
(def ^:dynamic *remove-nil?*
"When creating a dataset, omit any nil-valued fields from the documents."
@@ -69,20 +69,21 @@
[driver {:keys [table-definitions], :as dbdef} & {:keys [skip-drop-db?], :or {skip-drop-db? false}}]
(when-not skip-drop-db?
(destroy-db! driver dbdef))
- (with-mongo-connection [mongo-db (tx/dbdef->connection-details driver :db dbdef)]
+ (mongo.connection/with-mongo-database [^MongoDatabase db (tx/dbdef->connection-details driver :db dbdef)]
(doseq [{:keys [field-definitions table-name rows]} table-definitions]
(doseq [{:keys [field-name indexed?]} field-definitions]
(when indexed?
- (mcoll/create-index mongo-db table-name {field-name 1})))
+ (mongo.util/create-index (mongo.util/collection db table-name) {field-name 1})))
(let [field-names (for [field-definition field-definitions]
(keyword (:field-name field-definition)))]
;; Use map-indexed so we can get an ID for each row (index + 1)
(doseq [[i row] (map-indexed vector rows)]
(try
;; Insert each row
- (mcoll/insert mongo-db (name table-name) (into (ordered-map/ordered-map :_id (inc i))
- (cond->> (zipmap field-names row)
- *remove-nil?* (m/remove-vals nil?))))
+ (mongo.util/insert-one (mongo.util/collection db (name table-name))
+ (into (ordered-map/ordered-map :_id (inc i))
+ (cond->> (zipmap field-names row)
+ *remove-nil?* (m/remove-vals nil?))))
;; If row already exists then nothing to do
(catch com.mongodb.MongoException _)))))))
diff --git a/src/metabase/driver/util.clj b/src/metabase/driver/util.clj
index aec5c429438..91ce5b7fbe8 100644
--- a/src/metabase/driver/util.clj
+++ b/src/metabase/driver/util.clj
@@ -597,15 +597,20 @@
(.init trust-manager-factory trust-store)
(.getTrustManagers trust-manager-factory)))
-(defn ssl-socket-factory
- "Generates an `SocketFactory` with the custom certificates added"
- ^SocketFactory [& {:keys [private-key own-cert trust-cert]}]
+(defn ssl-context
+ "Generates a `SSLContext` with the custom certificates added."
+ ^javax.net.ssl.SSLContext [& {:keys [private-key own-cert trust-cert]}]
(let [ssl-context (SSLContext/getInstance "TLS")]
(.init ssl-context
(when (and private-key own-cert) (key-managers private-key (str (random-uuid)) own-cert))
(when trust-cert (trust-managers trust-cert))
nil)
- (.getSocketFactory ssl-context)))
+ ssl-context))
+
+(defn ssl-socket-factory
+ "Generates a `SocketFactory` with the custom certificates added."
+ ^SocketFactory [& {:keys [_private-key _own-cert _trust-cert] :as args}]
+ (.getSocketFactory (ssl-context args)))
(def default-sensitive-fields
"Set of fields that should always be obfuscated in API responses, as they contain sensitive data."
--
GitLab