diff --git a/src/metabase/api/database.clj b/src/metabase/api/database.clj
index 7fa4e2a8f06d841a0489868a5d9bab011571f127..5ae4a1bd70d72de6a56db38033eae784ef14578b 100644
--- a/src/metabase/api/database.clj
+++ b/src/metabase/api/database.clj
@@ -17,7 +17,7 @@
(defannotation DBEngine
"Param must be a valid database engine type, e.g. `h2` or `postgres`."
[symb value :nillable]
- (checkp-contains? (set (map name (keys driver/available-drivers))) symb value))
+ (checkp-contains? (set (map name (keys @driver/available-drivers))) symb value))
(defendpoint GET "/"
"Fetch all `Databases`."
@@ -46,7 +46,7 @@
"Values of options for the create/edit `Database` UI."
[]
{:timezones metabase.models.common/timezones
- :engines driver/available-drivers})
+ :engines @driver/available-drivers})
;; Stub function that will eventually validate a connection string
(defendpoint POST "/validate"
diff --git a/src/metabase/api/setup.clj b/src/metabase/api/setup.clj
index 42c1d1d5259203c7e7e9157f12ab662abadf8ad4..d24a375458ce8303d31162b6ce3ca0a91a4900c0 100644
--- a/src/metabase/api/setup.clj
+++ b/src/metabase/api/setup.clj
@@ -1,6 +1,7 @@
(ns metabase.api.setup
(:require [compojure.core :refer [defroutes POST]]
- [metabase.api.common :refer :all]
+ (metabase.api [common :refer :all]
+ [database :refer [annotation:DBEngine]])
[metabase.db :refer :all]
[metabase.driver :as driver]
[metabase.events :as events]
@@ -11,11 +12,6 @@
[metabase.setup :as setup]
[metabase.util :as u]))
-(defannotation DBEngine
- "Param must be a valid database engine type, e.g. `h2` or `postgres`."
- [symb value :nillable]
- (checkp-contains? (set (map name (keys driver/available-drivers))) symb value))
-
(defannotation SetupToken
"Check that param matches setup token or throw a 403."
[symb value]
diff --git a/src/metabase/driver.clj b/src/metabase/driver.clj
index 8c9d689facada85c5b4c736330b279b20ca632bf..0647ff370a9ca2b303b15498a199d0b3fa7de9af 100644
--- a/src/metabase/driver.clj
+++ b/src/metabase/driver.clj
@@ -2,10 +2,10 @@
(:require clojure.java.classpath
[clojure.string :as s]
[clojure.tools.logging :as log]
+ [clojure.tools.namespace.find :as ns-find]
[medley.core :as m]
[metabase.db :refer [ins sel upd]]
- (metabase.driver [interface :as i]
- [query-processor :as qp])
+ [metabase.driver.query-processor :as qp]
(metabase.models [database :refer [Database]]
[query-execution :refer [QueryExecution]])
[metabase.models.setting :refer [defsetting]]
@@ -13,30 +13,32 @@
(declare -dataset-query query-fail query-complete save-query-execution)
-;; ## CONFIG
+;;; ## CONFIG
(defsetting report-timezone "Connection timezone to use when executing queries. Defaults to system timezone.")
-
;; ## Constants
-(def ^:const available-drivers
- "Available DB drivers."
- {:h2 {:id "h2"
- :name "H2"}
- :postgres {:id "postgres"
- :name "Postgres"}
- :mongo {:id "mongo"
- :name "MongoDB"}
- :mysql {:id "mysql"
- :name "MySQL"}})
+(def available-drivers
+ "Delay to a map of info about available drivers."
+ (delay (->> (for [namespace (->> (ns-find/find-namespaces (clojure.java.classpath/classpath))
+ (filter (fn [ns-symb]
+ (re-matches #"^metabase\.driver\.[a-z0-9_]+$" (name ns-symb)))))]
+ (do (require namespace)
+ (->> (ns-publics namespace)
+ (map (fn [[symb varr]]
+ (when (::driver (meta varr))
+ {(keyword symb) (select-keys @varr [:details-fields
+ :driver-name
+ :features])})))
+ (into {}))))
+ (into {}))))
(defn is-engine?
- "Predicate function which validates if the given argument represents a valid driver identifier."
+ "Is ENGINE a valid driver name?"
[engine]
- (if (not (nil? engine))
- (contains? (set (map name (keys available-drivers))) (name engine))
- false))
+ (when engine
+ (contains? (set (keys @available-drivers)) (keyword engine))))
(defn class->base-type
"Return the `Field.base_type` that corresponds to a given class returned by the DB."
@@ -65,13 +67,11 @@
"Return the driver instance that should be used for given ENGINE.
This loads the corresponding driver if needed; it is expected that it resides in a var named
- metabase.driver.<engine>/driver"
+ metabase.driver.<engine>/<engine>"
[engine]
- {:pre [(keyword? engine)
- (contains? (set (keys available-drivers)) engine)]}
(let [nmspc (symbol (format "metabase.driver.%s" (name engine)))]
(require nmspc)
- @(ns-resolve nmspc 'driver)))
+ @(ns-resolve nmspc (symbol (name engine)))))
;; Can the type of a DB change?
@@ -93,11 +93,12 @@
"Check whether we can connect to DATABASE and perform a basic query (such as `SELECT 1`)."
[database]
{:pre [(map? database)]}
- (try
- (i/can-connect? (engine->driver (:engine database)) database)
- (catch Throwable e
- (log/error "Failed to connect to database:" (.getMessage e))
- false)))
+ (let [driver (engine->driver (:engine database))]
+ (try
+ ((:can-connect? driver) (:details database))
+ (catch Throwable e
+ (log/error "Failed to connect to database:" (.getMessage e))
+ false))))
(defn can-connect-with-details?
"Check whether we can connect to a database with ENGINE and DETAILS-MAP and perform a basic query.
@@ -107,15 +108,17 @@
(can-connect-with-details? :postgres {:host \"localhost\", :port 5432, ...})"
[engine details-map & [rethrow-exceptions]]
{:pre [(keyword? engine)
- (contains? (set (keys available-drivers)) engine)
+ (is-engine? engine)
(map? details-map)]}
- (let [driver (engine->driver engine)]
+ (let [{:keys [can-connect? humanize-connection-error-message]} (engine->driver engine)]
(try
- (i/can-connect-with-details? driver details-map)
+ (can-connect? details-map)
(catch Throwable e
(log/error "Failed to connect to database:" (.getMessage e))
(when rethrow-exceptions
- (let [message (i/humanize-connection-error-message driver (.getMessage e))]
+ (let [^String message ((or humanize-connection-error-message
+ identity)
+ (.getMessage e))]
(throw (Exception. message))))
false))))
diff --git a/src/metabase/driver/generic_sql.clj b/src/metabase/driver/generic_sql.clj
index de9df5ca5cc7e507526c03032a681b9aad684cc8..71a14c3b4dea576483c763168e566b70b902f4e6 100644
--- a/src/metabase/driver/generic_sql.clj
+++ b/src/metabase/driver/generic_sql.clj
@@ -4,19 +4,13 @@
[korma.core :as k]
[korma.sql.utils :as utils]
[metabase.driver :as driver]
- (metabase.driver [interface :refer [max-sync-lazy-seq-results IDriver ISyncDriverTableFKs ISyncDriverFieldAvgLength ISyncDriverFieldPercentUrls]]
+ (metabase.driver [interface :refer [max-sync-lazy-seq-results defdriver]]
[sync :as driver-sync])
- (metabase.driver.generic-sql [interface :as i]
- [query-processor :as qp]
+ (metabase.driver.generic-sql [query-processor :as qp]
[util :refer :all])
+ [metabase.models.field :as field]
[metabase.util :as u]))
-(def ^:const features
- "Features supported by *all* Generic SQL drivers."
- #{:foreign-keys
- :standard-deviation-aggregations
- :unix-timestamp-special-type-fields})
-
(def ^:private ^:const field-values-lazy-seq-chunk-size
"How many Field values should we fetch at a time for `field-values-lazy-seq`?"
;; Hopefully this is a good balance between
@@ -25,35 +19,28 @@
;; 3. Not fetching too many results for things like mark-json-field! which will fail after the first result that isn't valid JSON
500)
-(defn- can-connect-with-details? [driver details]
- (let [connection (i/connection-details->connection-spec driver details)]
+(defn- can-connect? [connection-details->spec details]
+ (let [connection (connection-details->spec details)]
(= 1 (-> (k/exec-raw connection "SELECT 1" :results)
first
vals
first))))
-(defn- can-connect? [driver database]
- (can-connect-with-details? driver (i/database->connection-details driver database)))
-
-(defn- wrap-process-query-middleware [_ qp]
- (fn [query]
- (qp query)))
-
-(defn- process-query [_ query]
+(defn- process-query [query]
(qp/process-and-run query))
-(defn- sync-in-context [_ database do-sync-fn]
+(defn- sync-in-context [database do-sync-fn]
(with-jdbc-metadata [_ database]
(do-sync-fn)))
-(defn- active-table-names [_ database]
+(defn- active-table-names [database]
(with-jdbc-metadata [^java.sql.DatabaseMetaData md database]
(->> (.getTables md nil nil nil (into-array String ["TABLE", "VIEW"]))
jdbc/result-set-seq
(map :table_name)
set)))
-(defn- active-column-names->type [{:keys [column->base-type]} table]
+(defn- active-column-names->type [column->base-type table]
{:pre [(map? column->base-type)]}
(with-jdbc-metadata [^java.sql.DatabaseMetaData md @(:db table)]
(->> (.getColumns md nil nil (:name table) nil)
@@ -64,14 +51,14 @@
:UnknownField)}))
(into {}))))
-(defn- table-pks [_ table]
+(defn- table-pks [table]
(with-jdbc-metadata [^java.sql.DatabaseMetaData md @(:db table)]
(->> (.getPrimaryKeys md nil nil (:name table))
jdbc/result-set-seq
(map :column_name)
set)))
-(defn- field-values-lazy-seq [_ {:keys [qualified-name-components table], :as field}]
+(defn- field-values-lazy-seq [{:keys [qualified-name-components table], :as field}]
(assert (and (map? field)
(delay? qualified-name-components)
(delay? table))
@@ -96,12 +83,12 @@
(fetch-chunk 0 field-values-lazy-seq-chunk-size
max-sync-lazy-seq-results)))
-(defn- table-rows-seq [_ database table-name]
+(defn- table-rows-seq [database table-name]
(k/select (-> (k/create-entity table-name)
(k/database (db->korma-db database)))))
-(defn- table-fks [_ table]
+(defn- table-fks [table]
(with-jdbc-metadata [^java.sql.DatabaseMetaData md @(:db table)]
(->> (.getImportedKeys md nil nil (:name table))
jdbc/result-set-seq
@@ -111,8 +98,7 @@
:dest-column-name (:pkcolumn_name result)}))
set)))
-(defn- field-avg-length [{:keys [sql-string-length-fn], :as driver} field]
- {:pre [(keyword? sql-string-length-fn)]}
+(defn- field-avg-length [sql-string-length-fn field]
(or (some-> (korma-entity @(:table field))
(k/select (k/aggregate (avg (k/sqlfn* sql-string-length-fn
(utils/func "CAST(%s AS CHAR)"
@@ -123,7 +109,7 @@
int)
0))
-(defn- field-percent-urls [_ field]
+(defn- field-percent-urls [field]
(or (let [korma-table (korma-entity @(:table field))]
(when-let [total-non-null-count (:count (first (k/select korma-table
(k/aggregate (count :*) :count)
@@ -135,31 +121,88 @@
(float (/ url-count total-non-null-count))))))
0.0))
-(def ^:const GenericSQLIDriverMixin
- "Generic SQL implementation of the `IDriver` protocol.
-
- (extend H2Driver
- IDriver
- GenericSQLIDriverMixin)"
- {:can-connect? can-connect?
- :can-connect-with-details? can-connect-with-details?
- :wrap-process-query-middleware wrap-process-query-middleware
- :process-query process-query
- :sync-in-context sync-in-context
- :active-table-names active-table-names
- :active-column-names->type active-column-names->type
- :table-pks table-pks
- :field-values-lazy-seq field-values-lazy-seq
- :table-rows-seq table-rows-seq})
-
-(def ^:const GenericSQLISyncDriverTableFKsMixin
- "Generic SQL implementation of the `ISyncDriverTableFKs` protocol."
- {:table-fks table-fks})
-
-(def ^:const GenericSQLISyncDriverFieldAvgLengthMixin
- "Generic SQL implementation of the `ISyncDriverFieldAvgLengthMixin` protocol."
- {:field-avg-length field-avg-length})
-
-(def ^:const GenericSQLISyncDriverFieldPercentUrlsMixin
- "Generic SQL implementation of the `ISyncDriverFieldPercentUrls` protocol."
- {:field-percent-urls field-percent-urls})
+(def ^:private ^:const required-fns
+ "Functions that concrete SQL drivers must define."
+ #{:connection-details->spec
+ :unix-timestamp->timestamp
+ :date
+ :date-interval})
+
+(defn- verify-sql-driver [{:keys [column->base-type sql-string-length-fn], :as driver}]
+ ;; Check the :column->base-type map
+ (assert column->base-type
+ "SQL drivers must define :column->base-type.")
+ (assert (map? column->base-type)
+ ":column->base-type should be a map")
+ (doseq [[k v] column->base-type]
+ (assert (keyword? k)
+ (format "Not a keyword: %s" k))
+ (assert (contains? field/base-types v)
+ (format "Invalid field base-type: %s" v)))
+
+ ;; Check :sql-string-length-fn
+ (assert sql-string-length-fn
+ "SQL drivers must define :sql-string-length-fn.")
+ (assert (keyword? sql-string-length-fn)
+ ":sql-string-length-fn must be a keyword.")
+
+ ;; Check required fns
+ (doseq [f required-fns]
+ (assert (f driver)
+ (format "SQL drivers must define %s." f))
+ (assert (fn? (f driver))
+ (format "%s must be a fn." f))))
+
+(defn sql-driver
+ "Create a Metabase DB driver using the Generic SQL functions.
+
+ A SQL driver must define the following properties / functions:
+
+ * `column->base-type`
+
+ A map of native DB column types (as keywords) to the `Field` `base-types` they map to.
+
+ * `sql-string-length-fn`
+
+ Keyword name of the SQL function that should be used to get the length of a string, e.g. `:LENGTH`.
+
+ * `(connection-details->spec [details-map])`
+
+ Given a `Database` DETAILS-MAP, return a JDBC connection spec.
+
+ * `(unix-timestamp->timestamp [seconds-or-milliseconds field-or-value])`
+
+ Return a korma form appropriate for converting a Unix timestamp integer field or value to an proper SQL `Timestamp`.
+ SECONDS-OR-MILLISECONDS refers to the resolution of the int in question and with be either `:seconds` or `:milliseconds`.
+
+ * `(timezone->set-timezone-sql [timezone])`
+
+ Return a string that represents the SQL statement that should be used to set the timezone
+ for the current transaction.
+
+ * `(date [this ^Keyword unit field-or-value])`
+
+ Return a korma form for truncating a date or timestamp field or value to a given resolution, or extracting a
+ date component.
+
+ * `(date-interval [unit amount])`
+
+ Return a korma form for a date relative to NOW(), e.g. on that would produce SQL like `(NOW() + INTERVAL '1 month')`."
+ [driver]
+ ;; Verify the driver
+ (verify-sql-driver driver)
+ (merge
+ {:features #{:foreign-keys
+ :standard-deviation-aggregations
+ :unix-timestamp-special-type-fields}
+ :can-connect? (partial can-connect? (:connection-details->spec driver))
+ :process-query process-query
+ :sync-in-context sync-in-context
+ :active-table-names active-table-names
+ :active-column-names->type (partial active-column-names->type (:column->base-type driver))
+ :table-pks table-pks
+ :field-values-lazy-seq field-values-lazy-seq
+ :table-rows-seq table-rows-seq
+ :table-fks table-fks
+ :field-avg-length (partial field-avg-length (:sql-string-length-fn driver))}
+ driver))
diff --git a/src/metabase/driver/generic_sql/interface.clj b/src/metabase/driver/generic_sql/interface.clj
deleted file mode 100644
index cf7913c2506dd865c05674b6d5d346006fd6c509..0000000000000000000000000000000000000000
--- a/src/metabase/driver/generic_sql/interface.clj
+++ /dev/null
@@ -1,26 +0,0 @@
-(ns metabase.driver.generic-sql.interface
- (:import clojure.lang.Keyword))
-
-(defprotocol ISqlDriverDatabaseSpecific
- "Methods a DB-specific concrete SQL driver should implement.
- They should also have the following properties:
-
- * `column->base-type`
- * `sql-string-length-fn`"
-
- (connection-details->connection-spec [this connection-details])
- (database->connection-details [this database])
-
- (unix-timestamp->timestamp [this ^Keyword seconds-or-milliseconds field-or-value]
- "Return a korma form appropriate for converting a Unix timestamp integer field or value to an proper SQL `Timestamp`.
- SECONDS-OR-MILLISECONDS refers to the resolution of the int in question and with be either `:seconds` or `:milliseconds`.")
-
- (timezone->set-timezone-sql [this timezone]
- "Return a string that represents the SQL statement that should be used to set the timezone
- for the current transaction.")
-
- (date [this ^Keyword unit field-or-value]
- "Return a korma form for truncating a date or timestamp field or value to a given resolution, or extracting a date component.")
-
- (date-interval [this ^Keyword unit ^Integer amount]
- "Return a korma form for a date relative to NOW(), e.g. on that would produce SQL like `(NOW() + INTERVAL '1 month')`."))
diff --git a/src/metabase/driver/generic_sql/native.clj b/src/metabase/driver/generic_sql/native.clj
index 28a95110ffd2f71640dfa480adacd66d9823036f..489a73ccddff25e280a32e90e9a835e07b3e266b 100644
--- a/src/metabase/driver/generic_sql/native.clj
+++ b/src/metabase/driver/generic_sql/native.clj
@@ -6,9 +6,7 @@
db)
[metabase.db :refer [sel]]
[metabase.driver :as driver]
- [metabase.driver.interface :refer [supports?]]
- (metabase.driver.generic-sql [interface :as i]
- [util :refer :all])
+ [metabase.driver.generic-sql.util :refer :all]
[metabase.models.database :refer [Database]]))
(defn- value->base-type
@@ -34,10 +32,10 @@
(when-let [timezone (or (-> query :native :timezone)
(driver/report-timezone))]
(when (seq timezone)
- (let [driver (driver/engine->driver (:engine database))]
- (when (supports? driver :set-timezone)
+ (let [{:keys [features timezone->set-timezone-sql]} (driver/engine->driver (:engine database))]
+ (when (contains? features :set-timezone)
(log/debug "Setting timezone to:" timezone)
- (jdbc/db-do-prepared conn (i/timezone->set-timezone-sql driver timezone))))))
+ (jdbc/db-do-prepared conn (timezone->set-timezone-sql timezone))))))
(jdbc/query conn sql :as-arrays? true))]
;; TODO - Why don't we just use annotate?
{:rows rows
diff --git a/src/metabase/driver/generic_sql/query_processor.clj b/src/metabase/driver/generic_sql/query_processor.clj
index bfe3b9e4f2cb25261ed03cb7c71453d76724608c..f2238d9ebf006ff4179191fcff46035d5c5ea9a0 100644
--- a/src/metabase/driver/generic_sql/query_processor.clj
+++ b/src/metabase/driver/generic_sql/query_processor.clj
@@ -8,10 +8,8 @@
[korma.sql.utils :as utils]
[metabase.config :as config]
[metabase.driver :as driver]
- (metabase.driver [interface :refer [supports?]]
- [query-processor :as qp])
- (metabase.driver.generic-sql [interface :as i]
- [native :as native]
+ [metabase.driver.query-processor :as qp]
+ (metabase.driver.generic-sql [native :as native]
[util :refer :all])
[metabase.util :as u])
(:import java.sql.Timestamp
@@ -42,9 +40,9 @@
(mapcat #(if (vector? %) % [%]))))
set-timezone-sql (when-let [timezone (driver/report-timezone)]
(when (seq timezone)
- (let [driver (:driver *query*)]
- (when (supports? driver :set-timezone)
- `(exec-raw ~(i/timezone->set-timezone-sql driver timezone))))))
+ (let [{:keys [features timezone->set-timezone-sql]} (:driver *query*)]
+ (when (contains? features :set-timezone)
+ `(exec-raw ~(timezone->set-timezone-sql timezone))))))
korma-form `(let [~'entity (korma-entity ~database ~source-table)]
~(if set-timezone-sql `(korma.db/with-db (:db ~'entity)
(korma.db/transaction
@@ -100,7 +98,7 @@
([this]
(formatted this false))
([{:keys [table-name special-type field-name], :as field} include-as?]
- (let [->timestamp (partial i/unix-timestamp->timestamp (:driver *query*))
+ (let [->timestamp (:unix-timestamp->timestamp (:driver *query*))
field (cond-> (keyword (str table-name \. field-name))
(= special-type :timestamp_seconds) (->timestamp :seconds)
(= special-type :timestamp_milliseconds) (->timestamp :milliseconds))]
@@ -112,7 +110,7 @@
([this]
(formatted this false))
([{unit :unit, {:keys [field-name base-type special-type], :as field} :field} include-as?]
- (let [field (i/date (:driver *query*) unit (formatted field))]
+ (let [field ((:date (:driver *query*)) unit (formatted field))]
(if include-as? [field (keyword field-name)]
field))))
@@ -145,7 +143,7 @@
(formatted this false))
([{value :value, {unit :unit} :field} _]
;; prevent Clojure from converting this to #inst literal, which is a util.date
- (i/date (:driver *query*) unit `(Timestamp/valueOf ~(.toString value)))))
+ ((:date (:driver *query*)) unit `(Timestamp/valueOf ~(.toString value)))))
RelativeDateTimeValue
(formatted
@@ -153,9 +151,9 @@
(formatted this false))
([{:keys [amount unit], {field-unit :unit} :field} _]
(let [driver (:driver *query*)]
- (i/date driver field-unit (if (zero? amount)
- (sqlfn :NOW)
- (i/date-interval driver unit amount)))))))
+ ((:date driver) field-unit (if (zero? amount)
+ (sqlfn :NOW)
+ ((:date-interval driver) unit amount)))))))
(defmethod apply-form :aggregation [[_ {:keys [aggregation-type field]}]]
diff --git a/src/metabase/driver/generic_sql/util.clj b/src/metabase/driver/generic_sql/util.clj
index fd4a1da838c05606650abb5071cb86dc18b35845..212330b9fad6d388d494c8ca41a753139c242251 100644
--- a/src/metabase/driver/generic_sql/util.clj
+++ b/src/metabase/driver/generic_sql/util.clj
@@ -7,19 +7,16 @@
[db :as kdb])
[korma.sql.utils :as utils]
[metabase.driver :as driver]
- [metabase.driver.query-processor :as qp]
- [metabase.driver.generic-sql.interface :as i]))
+ [metabase.driver.query-processor :as qp]))
(defn- db->connection-spec
"Return a JDBC connection spec for a Metabase `Database`."
[{{:keys [short-lived?]} :details, :as database}]
- (let [driver (driver/engine->driver (:engine database))
- database->connection-details (partial i/database->connection-details driver)
- connection-details->connection-spec (partial i/connection-details->connection-spec driver)]
- (merge (-> database database->connection-details connection-details->connection-spec)
+ (let [{:keys [connection-details->spec]} (driver/engine->driver (:engine database))]
+ (assoc (connection-details->spec (:details database))
;; unless this is a temp DB, we need to make a pool or the connection will be closed before we get a chance to unCLOB-er the results during JSON serialization
;; TODO - what will we do once we have CLOBS in temp DBs?
- {:make-pool? (not short-lived?)})))
+ :make-pool? (not short-lived?))))
(def ^{:arglists '([database])}
db->korma-db
diff --git a/src/metabase/driver/h2.clj b/src/metabase/driver/h2.clj
index 166b0aa7e7e19a5be8631b0103d3d780e61ae409..7219f041807ae613d02e3ea1c7eae4fb54790186 100644
--- a/src/metabase/driver/h2.clj
+++ b/src/metabase/driver/h2.clj
@@ -3,12 +3,9 @@
[korma.db :as kdb]
[korma.sql.utils :as utils]
[metabase.db :as db]
- [metabase.driver :as driver]
- (metabase.driver [generic-sql :as generic-sql, :refer [GenericSQLIDriverMixin GenericSQLISyncDriverTableFKsMixin
- GenericSQLISyncDriverFieldAvgLengthMixin GenericSQLISyncDriverFieldPercentUrlsMixin]]
- [interface :as i, :refer [IDriver ISyncDriverTableFKs ISyncDriverFieldAvgLength ISyncDriverFieldPercentUrls]])
- (metabase.driver.generic-sql [interface :refer [ISqlDriverDatabaseSpecific]]
- [util :refer [funcs]])
+ (metabase.driver [generic-sql :refer [sql-driver]]
+ [interface :as i, :refer [defdriver]])
+ [metabase.driver.generic-sql.util :refer [funcs]]
[metabase.models.database :refer [Database]]))
(def ^:private ^:const column->base-type
@@ -104,20 +101,17 @@
(file+options->connection-string file (merge options {"IFEXISTS" "TRUE"
"ACCESS_MODE_DATA" "r"}))))
-(defn- connection-details->connection-spec [_ details]
+(defn- connection-details->spec [details]
(kdb/h2 (if db/*allow-potentailly-unsafe-connections* details
(update details :db connection-string-set-safe-options))))
-(defn- database->connection-details [_ {:keys [details]}]
- details)
-
-(defn- unix-timestamp->timestamp [_ field-or-value seconds-or-milliseconds]
+(defn- unix-timestamp->timestamp [field-or-value seconds-or-milliseconds]
(utils/func (format "TIMESTAMPADD('%s', %%s, TIMESTAMP '1970-01-01T00:00:00Z')" (case seconds-or-milliseconds
:seconds "SECOND"
:milliseconds "MILLISECOND"))
[field-or-value]))
-(defn- wrap-process-query-middleware [_ qp]
+(defn- process-query-in-context [qp]
(fn [{query-type :type, :as query}]
{:pre [query-type]}
;; For :native queries check to make sure the DB in question has a (non-default) NAME property specified in the connection string.
@@ -153,7 +147,7 @@
["YEAR(%s)" field-or-value]
["((QUARTER(%s) * 3) - 2)" field-or-value]]))
-(defn- date [_ unit field-or-value]
+(defn- date [unit field-or-value]
(if (= unit :quarter)
(trunc-to-quarter field-or-value)
(utils/func (case unit
@@ -175,7 +169,7 @@
[field-or-value])))
;; TODO - maybe rename this relative-date ?
-(defn- date-interval [_ unit amount]
+(defn- date-interval [unit amount]
(utils/generated (format (case unit
:minute "DATEADD('MINUTE', %d, NOW())"
:hour "DATEADD('HOUR', %d, NOW())"
@@ -186,7 +180,7 @@
:year "DATEADD('YEAR', %d, NOW())")
amount)))
-(defn- humanize-connection-error-message [_ message]
+(defn- humanize-connection-error-message [message]
(condp re-matches message
#"^A file path that is implicitly relative to the current working directory is not allowed in the database URL .*$"
(i/connection-error-messages :cannot-connect-check-host-and-port)
@@ -200,24 +194,17 @@
#".*" ; default
message))
-
-(defrecord H2Driver [])
-
-(extend H2Driver
- ISqlDriverDatabaseSpecific {:connection-details->connection-spec connection-details->connection-spec
- :database->connection-details database->connection-details
- :date date
- :date-interval date-interval
- :unix-timestamp->timestamp unix-timestamp->timestamp}
- ;; Override the generic SQL implementation of wrap-process-query-middleware so we can block unsafe native queries (see above)
- IDriver (assoc GenericSQLIDriverMixin
- :humanize-connection-error-message humanize-connection-error-message
- :wrap-process-query-middleware wrap-process-query-middleware)
- ISyncDriverTableFKs GenericSQLISyncDriverTableFKsMixin
- ISyncDriverFieldAvgLength GenericSQLISyncDriverFieldAvgLengthMixin
- ISyncDriverFieldPercentUrls GenericSQLISyncDriverFieldPercentUrlsMixin)
-
-(def ^:const driver
- (map->H2Driver {:column->base-type column->base-type
- :features generic-sql/features
- :sql-string-length-fn :LENGTH}))
+(defdriver h2
+ (sql-driver {:driver-name "H2"
+ :details-fields [{:name "db"
+ :display-name "Connection String"
+ :placeholder "file:/Users/camsaul/bird_sightings/toucans;AUTO_SERVER=TRUE"
+ :required true}]
+ :column->base-type column->base-type
+ :sql-string-length-fn :LENGTH
+ :connection-details->spec connection-details->spec
+ :date date
+ :date-interval date-interval
+ :unix-timestamp->timestamp unix-timestamp->timestamp
+ :humanize-connection-error-message humanize-connection-error-message
+ :process-query-in-context process-query-in-context}))
diff --git a/src/metabase/driver/interface.clj b/src/metabase/driver/interface.clj
index 43d67bd7229ee3d625cce0665cd8bac29447eb1a..8baea6b53f4f620fcfefedb8a9a70dc5a53124be 100644
--- a/src/metabase/driver/interface.clj
+++ b/src/metabase/driver/interface.clj
@@ -1,14 +1,4 @@
-(ns metabase.driver.interface
- "Protocols that DB drivers implement. Thus, the interface such drivers provide."
- (:import (clojure.lang Keyword)))
-
-(def ^:const driver-optional-features
- "A set on optional features (as keywords) that may or may not be supported by individual drivers."
- #{:foreign-keys
- :nested-fields ; are nested Fields (i.e., Mongo-style nested keys) supported?
- :set-timezone
- :standard-deviation-aggregations
- :unix-timestamp-special-type-fields})
+(ns metabase.driver.interface)
(def ^:const max-sync-lazy-seq-results
"The maximum number of values we should return when using `field-values-lazy-seq`.
@@ -26,135 +16,225 @@
:username-incorrect "Looks like your username is incorrect."
:username-or-password-incorrect "Looks like the username or password is incorrect."})
-;; ## IDriver Protocol
+(def ^:private ^:const feature->required-fns
+ "Map of optional driver features (as keywords) to a set of functions drivers that support that feature must define."
+ {:foreign-keys #{:table-fks}
+ :nested-fields #{:active-nested-field-name->type}
+ :set-timezone nil
+ :standard-deviation-aggregations nil
+ :unix-timestamp-special-type-fields nil})
+
+(def ^:private ^:const optional-features
+ (set (keys feature->required-fns)))
+
+(def ^:private ^:const required-fns
+ #{:can-connect?
+ :active-table-names
+ :active-column-names->type
+ :table-pks
+ :field-values-lazy-seq
+ :process-query})
+
+(def ^:private ^:const optional-fns
+ #{:humanize-connection-error-message
+ :sync-in-context
+ :process-query-in-context
+ :table-rows-seq
+ :field-avg-length
+ :field-percent-urls
+ :driver-specific-sync-field!})
+
+(defn verify-driver
+ "Verify that a Metabase DB driver contains the expected properties and that they are the correct type."
+ [{:keys [driver-name details-fields features], :as driver}]
+ ;; Check :driver-name is a string
+ (assert driver-name
+ "Missing property :driver-name.")
+ (assert (string? driver-name)
+ ":driver-name must be a string.")
+
+ ;; Check the :details-fields
+ (assert details-fields
+ "Driver is missing property :details-fields.")
+ (assert (vector? details-fields)
+ ":details-fields should be a vector.")
+ (doseq [f details-fields]
+ (assert (map? f)
+ (format "Details fields must be maps: %s" f))
+ (assert (:name f)
+ (format "Details field %s is missing a :name property." f))
+ (assert (:display-name f)
+ (format "Details field %s is missing a :display-name property." f))
+ (when (:type f)
+ (assert (contains? #{:string :integer :password :boolean} (:type f))
+ (format "Invalid type %s in details field %s." (:type f) f)))
+ (when (:default f)
+ (assert (not (:placeholder f))
+ (format "Fields should not define both :default and :placeholder: %s" f))
+ (assert (not (:required f))
+ (format "Fields that define a :default cannot be :required: %s" f))))
-(defprotocol IDriver
- "Methods all drivers must implement.
- They should also include the following properties:
+ ;; Check that all required functions are defined
+ (doseq [f required-fns]
+ (assert (f driver)
+ (format "Missing fn: %s" f))
+ (assert (fn? (f driver))
+ (format "Not a fn: %s" (f driver))))
- * `features` (optional)
- A set containing one or more `driver-optional-features`"
+ ;; Check that all features declared are valid
+ (when features
+ (assert (and (set? features)
+ (every? keyword? features))
+ ":features must be a set of keywords.")
+ (doseq [feature features]
+ (assert (contains? optional-features feature)
+ (format "Not a valid feature: %s" feature))
+ (doseq [f (feature->required-fns feature)]
+ (assert (f driver)
+ (format "Drivers that support feature %s must have fn %s." feature f))
+ (assert (fn? (f driver))
+ (format "Not a fn: %s" f)))))
- ;; Connection
- (can-connect? [this database]
- "Check whether we can connect to DATABASE and perform a simple query.
- (To check whether we can connect to a database given only its details, use `can-connect-with-details?` instead).
+ ;; Check that the optional fns, if included, are actually fns
+ (doseq [f optional-fns]
+ (when (f driver)
+ (assert (fn? (f driver))
+ (format "Not a fn: %s" f)))))
- (can-connect? driver (sel :one Database :id 1))")
- (can-connect-with-details? [this details-map]
- "Check whether we can connect to a database and performa a simple query.
- Returns true if we can, otherwise returns `false` or throws an `Exception`.
+(defmacro defdriver
+ "Define and validate a new Metabase DB driver.
- (can-connect-with-details? driver {:engine :postgres, :dbname \"book\", ...})")
+ All drivers must include the following keys:
- (humanize-connection-error-message ^String [this ^String message]
- "Return a humanized (user-facing) version of an connection error message string.
- Generic error messages are provided in the constant `connection-error-messages`; return one of these whenever possible.")
+#### PROPERTIES
- ;; Syncing
- (sync-in-context [this database do-sync-fn]
- "This function is basically around-advice for `sync-database!` and `sync-table!` operations.
- Implementers can setup any context necessary for syncing, then need to call DO-SYNC-FN,
- which takes no args.
+* `:driver-name`
- (sync-in-context [_ database do-sync-fn]
+ A human-readable string naming the DB this driver works with, e.g. `\"PostgreSQL\"`.
+
+* `:details-fields`
+
+ A vector of maps that contain information about connection properties that should
+ be exposed to the user for databases that will use this driver. This information is used to build the UI for editing
+ a `Database` `details` map, and for validating it on the Backend. It should include things like `host`,
+ `port`, and other driver-specific parameters. Each field information map should have the following properties:
+
+ * `:name`
+
+ The key that should be used to store this property in the `details` map.
+
+ * `:display-name`
+
+ Human-readable name that should be displayed to the User in UI for editing this field.
+
+ * `:type` *(OPTIONAL)*
+
+ `:string`, `:integer`, `:boolean`, or `:password`. Defaults to `:string`.
+
+ * `:default` *(OPTIONAL)*
+
+ A default value for this field if the user hasn't set an explicit value. This is shown in the UI as a placeholder.
+
+ * `:placeholder` *(OPTIONAL)*
+
+ Placeholder value to show in the UI if user hasn't set an explicit value. Similar to `:default`, but this value is
+ *not* saved to `:details` if no explicit value is set. Since `:default` values are also shown as placeholders, you
+ cannot specify both `:default` and `:placeholder`.
+
+ * `:required` *(OPTIONAL)*
+
+ Is this property required? Defaults to `false`.
+
+* `:features` *(OPTIONAL)*
+
+ A set of keyword names of optional features supported by this driver, such as `:foreign-keys`.
+
+#### FUNCTIONS
+
+* `(can-connect? [details-map])`
+
+ Check whether we can connect to a `Database` with DETAILS-MAP and perform a simple query. For example, a SQL database might
+ try running a query like `SELECT 1;`. This function should return `true` or `false`.
+
+* `(active-table-names [database])`
+
+ Return a set of string names of tables, collections, or equivalent that currently exist in DATABASE.
+
+* `(active-column-names->type [table])`
+
+ Return a map of string names of active columns (or equivalent) -> `Field` `base_type` for TABLE (or equivalent).
+
+* `(table-pks [table])`
+
+ Return a set of string names of active Fields that are primary keys for TABLE (or equivalent).
+
+* `(field-values-lazy-seq [field])`
+
+ Return a lazy sequence of all values of FIELD.
+ This is used to implement `mark-json-field!`, and fallback implentations of `mark-no-preview-display-field!` and `mark-url-field!`
+ if drivers *don't* implement `field-avg-length` and `field-percent-urls`, respectively.
+
+* `(process-query [query])`
+
+ Process a native or structured QUERY. This function is called by `metabase.driver/process-query` after performing various driver-unspecific
+ steps like Query Expansion and other preprocessing.
+
+* `(table-fks [table])` *(REQUIRED FOR DRIVERS THAT SUPPORT `:foreign-keys`)*
+
+ Return a set of maps containing info about FK columns for TABLE.
+ Each map should contain the following keys:
+
+ * `fk-column-name`
+ * `dest-table-name`
+ * `dest-column-name`
+
+* `(active-nested-field-name->type [field])` *(REQUIRED FOR DRIVERS THAT SUPPORT `:nested-fields`)*
+
+ Return a map of string names of active child `Fields` of FIELD -> `Field.base_type`.
+
+* `(humanize-connection-error-message [message])` *(OPTIONAL)*
+
+ Return a humanized (user-facing) version of an connection error message string.
+ Generic error messages are provided in the constant `connection-error-messages`; return one of these whenever possible.
+
+* `(sync-in-context [database f])` *(OPTIONAL)*
+
+ Drivers may provide this function if they need to do special setup before a sync operation such as `sync-database!`. The sync
+ operation itself is encapsulated as the lambda F, which must be called with no arguments.
+
+ (defn sync-in-context [database f]
(with-jdbc-metadata [_ database]
- (do-sync-fn)))")
- (active-table-names [this database]
- "Return a set of string names of tables, collections, or equivalent that currently exist in DATABASE.")
- (active-column-names->type [this table]
- "Return a map of string names of active columns (or equivalent) -> `Field` `base_type` for TABLE (or equivalent).")
- (table-pks [this table]
- "Return a set of string names of active Fields that are primary keys for TABLE (or equivalent).")
- (field-values-lazy-seq [this field]
- "Return a lazy sequence of all values of Field.
- This is used to implement `mark-json-field!`, and fallback implentations of `mark-no-preview-display-field!` and `mark-url-field!`
- if drivers *don't* implement `ISyncDriverFieldAvgLength` or `ISyncDriverFieldPercentUrls`, respectively.")
- (table-rows-seq [this database table-name]
- "Return a sequence of all the rows in a table with a given TABLE-NAME.
- Currently, this is only used for iterating over the values in a `_metabase_metadata` table. As such, the results are not expected to be returned lazily.
-
- (table-rows-seq driver (Database 2) \"_metabase_metadata\")
- -> [{:keypath \"people.description\"
- :value \"...\"}
- ...]")
-
- ;; Query Processing
- (process-query [this query]
- "Process a native or structured query.
- (Don't use this directly; instead, use `metabase.driver/process-query`,
- which does things like preprocessing before calling the appropriate implementation.)")
- (wrap-process-query-middleware [this qp-fn]
- "Custom QP middleware for this driver.
- Like `sync-in-context`, but for running queries rather than syncing. This is basically around-advice for the QP pre and post-processing stages.
- This should be used to do things like open DB connections that need to remain open for the duration of post-processing.
- This middleware is injected into the QP middleware stack immediately after the Query Expander; in other words, it will receive the expanded query.
- See the Mongo driver for and example of how this is intended to be used."))
-
-
-;; ## ISyncDriverTableFKs Protocol (Optional)
-
-(defprotocol ISyncDriverTableFKs
- "Optional protocol to provide FK information for a TABLE.
- If a sync driver implements it, Table FKs will be synced; otherwise, the step will be skipped."
- (table-fks [this table]
- "Return a set of maps containing info about FK columns for TABLE.
- Each map should contain the following keys:
-
- * fk-column-name
- * dest-table-name
- * dest-column-name"))
-
-
-(defprotocol ISyncDriverFieldNestedFields
- "Optional protocol that should provide information about the subfields of a FIELD when applicable.
- Drivers that declare support for `:nested-fields` should implement this protocol."
- (active-nested-field-name->type [this field]
- "Return a map of string names of active child `Fields` of FIELD -> `Field.base_type`."))
-
-
-;; ## ISyncDriverField Protocols (Optional)
-
-;; These are optional protocol that drivers can implement to be used instead of falling back to field-values-lazy-seq for certain Field
-;; syncing operations, which involves iterating over a few thousand values of the Field in Clojure-land. Since that's slower, it's
-;; preferable to provide implementations of ISyncDriverFieldAvgLength/ISyncDriverFieldPercentUrls when possible.
-
-(defprotocol ISyncDriverFieldAvgLength
- "Optional. If this isn't provided, a fallback implementation that calculates average length in Clojure-land will be used instead."
- (field-avg-length [this field]
- "Return the average length of all non-nil values of textual FIELD."))
-
-(defprotocol ISyncDriverFieldPercentUrls
- "Optional. If this isn't provided, a fallback implementation that calculates URL percentage in Clojure-land will be used instead."
- (field-percent-urls [this field]
- "Return the percentage of non-nil values of textual FIELD that are valid URLs."))
-
-
-;;; ## ISyncDriverSpecificSyncField (Optional)
-
-(defprotocol ISyncDriverSpecificSyncField
- "Optional. Do driver-specific syncing for a FIELD."
- (driver-specific-sync-field! [this field]
- "This is a chance for drivers to do custom Field syncing specific to their database.
- For example, the Postgres driver can mark Postgres JSON fields as `special_type = json`.
- As with the other Field syncing functions in `metabase.driver.sync`, this method should return the modified
- FIELD, if any, or `nil`."))
-
-
-;; ## Helper Functions
-
-(def ^:private valid-feature? (partial contains? driver-optional-features))
-
-(defn supports?
- "Does DRIVER support FEATURE?"
- [{:keys [features]} ^Keyword feature]
- {:pre [(set? features)
- (every? valid-feature? features)
- (valid-feature? feature)]}
- (contains? features feature))
-
-(defn assert-driver-supports
- "Helper fn. Assert that DRIVER supports FEATURE."
- [driver ^Keyword feature]
- (when-not (supports? driver feature)
- (throw (Exception. (format "%s is not supported by this driver." (name feature))))))
+ (f)))
+
+* `(process-query-in-context [f])` *(OPTIONAL)*
+
+ Similar to `sync-in-context`, but for running queries rather than syncing. This should be used to do things like open DB connections
+ that need to remain open for the duration of post-processing. This function follows a middleware pattern and is injected into the QP
+ middleware stack immediately after the Query Expander; in other words, it will receive the expanded query.
+ See the Mongo and H2 drivers for examples of how this is intended to be used.
+
+* `(table-rows-seq [database table-name])` *(OPTIONAL)*
+
+ Return a sequence of all the rows in a table with a given TABLE-NAME.
+ Currently, this is only used for iterating over the values in a `_metabase_metadata` table. As such, the results are not expected to be returned lazily.
+
+* `(field-avg-length [field])` *(OPTIONAL)*
+
+ If possible, provide an efficent DB-level function to calculate the average length of non-nil values of textual FIELD, which is used to determine whether a `Field`
+ should be marked as a `:category`. If this function is not provided, a fallback implementation that iterates over results in Clojure-land is used instead.
+
+* `(field-percent-urls [field])` *(OPTIONAL)*
+
+ If possible, provide an efficent DB-level function to calculate what percentage of non-nil values of textual FIELD are valid URLs, which is used to determine
+ whether a `Field` should be marked as a `:url`. If this function is not provided, a fallback implementation that iterates over results in Clojure-land is used instead.
+
+* `(driver-specific-sync-field! [field])` *(OPTIONAL)*
+
+ This is a chance for drivers to do custom `Field` syncing specific to their database.
+ For example, the Postgres driver can mark Postgres JSON fields as `special_type = json`.
+ As with the other Field syncing functions in `metabase.driver.sync`, this method should return the modified FIELD, if any, or `nil`."
+ [driver-name driver-map]
+ `(def ~(vary-meta driver-name assoc :metabase.driver/driver (keyword driver-name))
+ (let [m# ~driver-map]
+ (verify-driver m#)
+ m#)))
diff --git a/src/metabase/driver/mongo.clj b/src/metabase/driver/mongo.clj
index a83778fc3066886147d479557fb1846a80e3c94d..33c776e4911f9e703365a4bb8d21d46cc1818265 100644
--- a/src/metabase/driver/mongo.clj
+++ b/src/metabase/driver/mongo.clj
@@ -12,7 +12,7 @@
[db :as mdb]
[query :as mq])
[metabase.driver :as driver]
- [metabase.driver.interface :as i, :refer [IDriver ISyncDriverFieldNestedFields]]
+ [metabase.driver.interface :as i, :refer [defdriver]]
(metabase.driver.mongo [query-processor :as qp]
[util :refer [*mongo-connection* with-mongo-connection values->base-type]])
[metabase.util :as u]))
@@ -38,22 +38,19 @@
{:pre [(map? field)]
:post [(keyword? %)]}
(with-mongo-connection [_ @(:db @(:table field))]
- (values->base-type (field-values-lazy-seq driver field))))
+ (values->base-type (field-values-lazy-seq field))))
;;; ## MongoDriver
-(defn- can-connect? [_ database]
- (with-mongo-connection [^com.mongodb.DB conn database]
+(defn- can-connect? [details]
+ (with-mongo-connection [^com.mongodb.DB conn details]
(= (-> (cmd/db-stats conn)
(conv/from-db-object :keywordize)
:ok)
1.0)))
-(defn- can-connect-with-details? [this details]
- (can-connect? this {:details details}))
-
-(defn- humanize-connection-error-message [_ message]
+(defn- humanize-connection-error-message [message]
(condp re-matches message
#"^Timed out after \d+ ms while waiting for a server .*$"
(i/connection-error-messages :cannot-connect-check-host-and-port)
@@ -64,28 +61,28 @@
#"^Password can not be null when the authentication mechanism is unspecified$"
(i/connection-error-messages :password-required)
- #".*" ; default
+ #".*" ; default
message))
-(defn- wrap-process-query-middleware [_ qp]
+(defn- process-query-in-context [qp]
(fn [query]
(with-mongo-connection [^com.mongodb.DB conn (:database query)]
(qp query))))
-(defn- process-query [_ query]
+(defn- process-query [query]
(qp/process-and-run query))
;;; ### Syncing
-(defn- sync-in-context [_ database do-sync-fn]
+(defn- sync-in-context [database do-sync-fn]
(with-mongo-connection [_ database]
(do-sync-fn)))
-(defn- active-table-names [_ database]
+(defn- active-table-names [database]
(with-mongo-connection [^com.mongodb.DB conn database]
(-> (mdb/get-collection-names conn)
(set/difference #{"system.indexes"}))))
-(defn- active-column-names->type [_ table]
+(defn- active-column-names->type [table]
(with-mongo-connection [_ @(:db table)]
(into {} (for [column-name (table->column-names table)]
{(name column-name)
@@ -93,7 +90,7 @@
:table (delay table)
:qualified-name-components (delay [(:name table) (name column-name)])})}))))
-(defn- field-values-lazy-seq [_ {:keys [qualified-name-components table], :as field}]
+(defn- field-values-lazy-seq [{:keys [qualified-name-components table], :as field}]
(assert (and (map? field)
(delay? qualified-name-components)
(delay? table))
@@ -108,11 +105,11 @@
(mq/with-collection *mongo-connection* (:name table)
(mq/fields [(apply str (interpose "." name-components))]))))))
-(defn- active-nested-field-name->type [this field]
+(defn- active-nested-field-name->type [field]
;; Build a map of nested-field-key -> type -> count
;; TODO - using an atom isn't the *fastest* thing in the world (but is the easiest); consider alternate implementation
(let [field->type->count (atom {})]
- (doseq [val (take i/max-sync-lazy-seq-results (field-values-lazy-seq this field))]
+ (doseq [val (take i/max-sync-lazy-seq-results (field-values-lazy-seq field))]
(when (map? val)
(doseq [[k v] val]
(swap! field->type->count update-in [k (type v)] #(if % (inc %) 1)))))
@@ -125,22 +122,33 @@
first ; keep just the type
driver/class->base-type))))))
-
-(defrecord MongoDriver [])
-
-(extend MongoDriver
- IDriver {:can-connect? can-connect?
- :can-connect-with-details? can-connect-with-details?
- :humanize-connection-error-message humanize-connection-error-message
- :wrap-process-query-middleware wrap-process-query-middleware
- :process-query process-query
- :sync-in-context sync-in-context
- :active-table-names active-table-names
- :active-column-names->type active-column-names->type
- :table-pks (constantly #{"_id"})
- :field-values-lazy-seq field-values-lazy-seq}
- ISyncDriverFieldNestedFields {:active-nested-field-name->type active-nested-field-name->type})
-
-(def driver
- "Concrete instance of the MongoDB driver."
- (map->MongoDriver {:features #{:nested-fields}}))
+(defdriver mongo
+ {:driver-name "MongoDB"
+ :details-fields [{:name "host"
+ :display-name "Host"
+ :default "localhost"}
+ {:name "port"
+ :display-name "Port"
+ :type :integer
+ :default 27017}
+ {:name "dbname"
+ :display-name "Database name"
+ :placeholder "carrierPigeonDeliveries"
+ :required true}
+ {:name "user"
+ :display-name "Database username"
+ :placeholder "What username do you use to login to the database?"}
+ {:name "pass"
+ :display-name "Database password"
+ :type :password
+ :placeholder "******"}]
+ :features #{:nested-fields}
+ :can-connect? can-connect?
+ :active-table-names active-table-names
+ :field-values-lazy-seq field-values-lazy-seq
+ :active-column-names->type active-column-names->type
+ :table-pks (constantly #{"_id"})
+ :process-query process-query
+ :process-query-in-context process-query-in-context
+ :sync-in-context sync-in-context
+ :active-nested-field-name->type active-nested-field-name->type})
diff --git a/src/metabase/driver/mongo/query_processor.clj b/src/metabase/driver/mongo/query_processor.clj
index 7a76f7e48901088c29eb42201ae93403f34af20c..e26addf607cbb36aa153333956cf178383720598 100644
--- a/src/metabase/driver/mongo/query_processor.clj
+++ b/src/metabase/driver/mongo/query_processor.clj
@@ -12,9 +12,7 @@
[operators :refer :all]
[query :refer :all])
[metabase.db :refer :all]
- [metabase.driver :as driver]
- (metabase.driver [interface :as i]
- [query-processor :as qp])
+ [metabase.driver.query-processor :as qp]
[metabase.driver.query-processor.interface :refer [qualified-name-components]]
[metabase.driver.mongo.util :refer [with-mongo-connection *mongo-connection* values->base-type]]
[metabase.models.field :as field]
@@ -162,7 +160,7 @@
(constantly true))
field-id (or (:field-id field) ; Field
(:field-id (:field field)))] ; DateTimeField
- (->> (i/field-values-lazy-seq @(ns-resolve 'metabase.driver.mongo 'driver) (sel :one field/Field :id field-id)) ; resolve driver at runtime to avoid circular deps
+ (->> (@(resolve 'metabase.driver.mongo/field-values-lazy-seq) (sel :one field/Field :id field-id)) ; resolve driver at runtime to avoid circular deps
(filter identity)
(map hash)
(map #(conj! values %))
diff --git a/src/metabase/driver/mysql.clj b/src/metabase/driver/mysql.clj
index 627ec5aa6a5c9733aa7afa57ac9b40627bc79afa..3a1a3845b90660ee5b289f0f37dd601ed4738273 100644
--- a/src/metabase/driver/mysql.clj
+++ b/src/metabase/driver/mysql.clj
@@ -6,12 +6,9 @@
mysql)
(korma.sql [engine :refer [sql-func]]
[utils :as utils])
- (metabase.driver [generic-sql :as generic-sql, :refer [GenericSQLIDriverMixin GenericSQLISyncDriverTableFKsMixin
- GenericSQLISyncDriverFieldAvgLengthMixin GenericSQLISyncDriverFieldPercentUrlsMixin]]
- [interface :as i, :refer [IDriver ISyncDriverTableFKs ISyncDriverFieldAvgLength ISyncDriverFieldPercentUrls
- ISyncDriverSpecificSyncField driver-specific-sync-field!]])
- (metabase.driver.generic-sql [interface :refer [ISqlDriverDatabaseSpecific]]
- [util :refer [funcs]])))
+ (metabase.driver [generic-sql :refer [sql-driver]]
+ [interface :as i, :refer [defdriver]])
+ [metabase.driver.generic-sql.util :refer [funcs]]))
;;; # Korma 0.4.2 Bug Workaround
;; (Buggy code @ https://github.com/korma/Korma/blob/684178c386df529558bbf82097635df6e75fb339/src/korma/mysql.clj)
@@ -61,21 +58,18 @@
:VARCHAR :TextField
:YEAR :IntegerField})
-(defn- connection-details->connection-spec [_ details]
+(defn- connection-details->spec [details]
(-> details
(set/rename-keys {:dbname :db})
kdb/mysql))
-(defn- database->connection-details [_ {:keys [details]}]
- details)
-
-(defn- unix-timestamp->timestamp [_ field-or-value seconds-or-milliseconds]
+(defn- unix-timestamp->timestamp [field-or-value seconds-or-milliseconds]
(utils/func (case seconds-or-milliseconds
:seconds "FROM_UNIXTIME(%s)"
:milliseconds "FROM_UNIXTIME(%s / 1000)")
[field-or-value]))
-(defn- timezone->set-timezone-sql [_ timezone]
+(defn- timezone->set-timezone-sql [timezone]
;; If this fails you need to load the timezone definitions from your system into MySQL;
;; run the command `mysql_tzinfo_to_sql /usr/share/zoneinfo | mysql -u root mysql`
;; See https://dev.mysql.com/doc/refman/5.7/en/time-zone-support.html for details
@@ -97,7 +91,7 @@
["((QUARTER(%s) * 3) - 2)" field-or-value]
(k/raw "'-01'")]))
-(defn- date [_ unit field-or-value]
+(defn- date [unit field-or-value]
(if (= unit :quarter)
(trunc-to-quarter field-or-value)
(utils/func (case unit
@@ -121,7 +115,7 @@
:year "YEAR(%s)")
[field-or-value])))
-(defn- date-interval [_ unit amount]
+(defn- date-interval [unit amount]
(utils/generated (format (case unit
:minute "DATE_ADD(NOW(), INTERVAL %d MINUTE)"
:hour "DATE_ADD(NOW(), INTERVAL %d HOUR)"
@@ -132,7 +126,7 @@
:year "DATE_ADD(NOW(), INTERVAL %d YEAR)")
amount)))
-(defn- humanize-connection-error-message [_ message]
+(defn- humanize-connection-error-message [message]
(condp re-matches message
#"^Communications link failure\s+The last packet sent successfully to the server was 0 milliseconds ago. The driver has not received any packets from the server.$"
(i/connection-error-messages :cannot-connect-check-host-and-port)
@@ -149,22 +143,34 @@
#".*" ; default
message))
-(defrecord MySQLDriver [])
-
-(extend MySQLDriver
- ISqlDriverDatabaseSpecific {:connection-details->connection-spec connection-details->connection-spec
- :database->connection-details database->connection-details
- :unix-timestamp->timestamp unix-timestamp->timestamp
- :date date
- :date-interval date-interval
- :timezone->set-timezone-sql timezone->set-timezone-sql}
- IDriver (assoc GenericSQLIDriverMixin
- :humanize-connection-error-message humanize-connection-error-message)
- ISyncDriverTableFKs GenericSQLISyncDriverTableFKsMixin
- ISyncDriverFieldAvgLength GenericSQLISyncDriverFieldAvgLengthMixin
- ISyncDriverFieldPercentUrls GenericSQLISyncDriverFieldPercentUrlsMixin)
-
-(def ^:const driver
- (map->MySQLDriver {:column->base-type column->base-type
- :features (conj generic-sql/features :set-timezone)
- :sql-string-length-fn :CHAR_LENGTH}))
+(defdriver mysql
+ (-> {:driver-name "MySQL"
+ :details-fields [{:name "host"
+ :display-name "Host"
+ :default "localhost"}
+ {:name "port"
+ :display-name "Port"
+ :type :integer
+ :default 3306}
+ {:name "dbname"
+ :display-name "Database name"
+ :placeholder "birds_of_the_word"
+ :required true}
+ {:name "user"
+ :display-name "Database username"
+ :placeholder "What username do you use to login to the database?"
+ :required true}
+ {:name "password"
+ :display-name "Database password"
+ :type :password
+ :placeholder "*******"}]
+ :column->base-type column->base-type
+ :sql-string-length-fn :CHAR_LENGTH
+ :connection-details->spec connection-details->spec
+ :unix-timestamp->timestamp unix-timestamp->timestamp
+ :date date
+ :date-interval date-interval
+ :timezone->set-timezone-sql timezone->set-timezone-sql
+ :humanize-connection-error-message humanize-connection-error-message}
+ sql-driver
+ (update :features conj :set-timezone)))
diff --git a/src/metabase/driver/postgres.clj b/src/metabase/driver/postgres.clj
index f21085a947b6530ae827fb259f0da8c7f12b3910..66660c8cbcd049f2c30c840a8ccd251393ae2182 100644
--- a/src/metabase/driver/postgres.clj
+++ b/src/metabase/driver/postgres.clj
@@ -9,14 +9,9 @@
[swiss.arrows :refer :all]
[metabase.db :refer [upd]]
[metabase.models.field :refer [Field]]
- [metabase.driver :as driver]
- (metabase.driver [generic-sql :as generic-sql, :refer [GenericSQLIDriverMixin GenericSQLISyncDriverTableFKsMixin
- GenericSQLISyncDriverFieldAvgLengthMixin GenericSQLISyncDriverFieldPercentUrlsMixin]]
- [interface :as i, :refer [IDriver ISyncDriverTableFKs ISyncDriverFieldAvgLength ISyncDriverFieldPercentUrls
- ISyncDriverSpecificSyncField]])
- [metabase.driver.generic-sql :as generic-sql]
- (metabase.driver.generic-sql [interface :refer [ISqlDriverDatabaseSpecific]]
- [util :refer [with-jdbc-metadata]]))
+ (metabase.driver [generic-sql :refer [sql-driver]]
+ [interface :as i, :refer [defdriver]])
+ [metabase.driver.generic-sql.util :refer [with-jdbc-metadata]])
;; This is necessary for when NonValidatingFactory is passed in the sslfactory connection string argument,
;; e.x. when connecting to a Heroku Postgres database from outside of Heroku.
(:import org.postgresql.ssl.NonValidatingFactory))
@@ -88,34 +83,28 @@
:sslmode "require"
:sslfactory "org.postgresql.ssl.NonValidatingFactory"}) ; HACK Why enable SSL if we disable certificate validation?
-(defn- connection-details->connection-spec [_ {:keys [ssl] :as details-map}]
+(defn- connection-details->spec [{:keys [ssl] :as details-map}]
(-> details-map
+ (update :port (fn [port]
+ (if (string? port) (Integer/parseInt port)
+ port)))
(dissoc :ssl) ; remove :ssl in case it's false; DB will still try (& fail) to connect if the key is there
(merge (when ssl ; merging ssl-params will add :ssl back in if desirable
ssl-params))
(rename-keys {:dbname :db})
kdb/postgres))
-(defn- database->connection-details [_ {:keys [details]}]
- (let [{:keys [host port]} details]
- (-> details
- (assoc :host host
- :ssl (:ssl details)
- :port (if (string? port) (Integer/parseInt port)
- port))
- (rename-keys {:dbname :db}))))
-
-(defn- unix-timestamp->timestamp [_ field-or-value seconds-or-milliseconds]
+(defn- unix-timestamp->timestamp [field-or-value seconds-or-milliseconds]
(utils/func (case seconds-or-milliseconds
:seconds "TO_TIMESTAMP(%s)"
:milliseconds "TO_TIMESTAMP(%s / 1000)")
[field-or-value]))
-(defn- timezone->set-timezone-sql [_ timezone]
+(defn- timezone->set-timezone-sql [timezone]
(format "SET LOCAL timezone TO '%s';" timezone))
-(defn- driver-specific-sync-field! [_ {:keys [table], :as field}]
+(defn- driver-specific-sync-field! [{:keys [table], :as field}]
(with-jdbc-metadata [^java.sql.DatabaseMetaData md @(:db @table)]
(let [[{:keys [type_name]}] (->> (.getColumns md nil nil (:name @table) (:name field))
jdbc/result-set-seq)]
@@ -123,7 +112,7 @@
(upd Field (:id field) :special_type :json)
(assoc field :special_type :json)))))
-(defn- date [_ unit field-or-value]
+(defn- date [unit field-or-value]
(utils/func (case unit
:default "CAST(%s AS TIMESTAMP)"
:minute "DATE_TRUNC('minute', %s)"
@@ -145,7 +134,7 @@
:year "CAST(EXTRACT(YEAR FROM %s) AS INTEGER)")
[field-or-value]))
-(defn- date-interval [_ unit amount]
+(defn- date-interval [unit amount]
(utils/generated (format (case unit
:minute "(NOW() + INTERVAL '%d minute')"
:hour "(NOW() + INTERVAL '%d hour')"
@@ -156,7 +145,7 @@
:year "(NOW() + INTERVAL '%d year')")
amount)))
-(defn- humanize-connection-error-message [_ message]
+(defn- humanize-connection-error-message [message]
(condp re-matches message
#"^FATAL: database \".*\" does not exist$"
(i/connection-error-messages :database-name-incorrect)
@@ -180,23 +169,39 @@
#".*" ; default
message))
-(defrecord PostgresDriver [])
-
-(extend PostgresDriver
- ISqlDriverDatabaseSpecific {:connection-details->connection-spec connection-details->connection-spec
- :database->connection-details database->connection-details
- :unix-timestamp->timestamp unix-timestamp->timestamp
- :date date
- :date-interval date-interval
- :timezone->set-timezone-sql timezone->set-timezone-sql}
- ISyncDriverSpecificSyncField {:driver-specific-sync-field! driver-specific-sync-field!}
- IDriver (assoc GenericSQLIDriverMixin
- :humanize-connection-error-message humanize-connection-error-message)
- ISyncDriverTableFKs GenericSQLISyncDriverTableFKsMixin
- ISyncDriverFieldAvgLength GenericSQLISyncDriverFieldAvgLengthMixin
- ISyncDriverFieldPercentUrls GenericSQLISyncDriverFieldPercentUrlsMixin)
-
-(def ^:const driver
- (map->PostgresDriver {:column->base-type column->base-type
- :features (conj generic-sql/features :set-timezone)
- :sql-string-length-fn :CHAR_LENGTH}))
+(defdriver postgres
+ (-> {:driver-name "PostgreSQL"
+ :details-fields [{:name "host"
+ :display-name "Host"
+ :default "localhost"}
+ {:name "port"
+ :display-name "Port"
+ :type :integer
+ :default 5432}
+ {:name "dbname"
+ :display-name "Database name"
+ :placeholder "birds_of_the_word"
+ :required true}
+ {:name "user"
+ :display-name "Database username"
+ :placeholder "What username do you use to login to the database?"
+ :required true}
+ {:name "password"
+ :display-name "Database password"
+ :type :password
+ :placeholder "*******"}
+ {:name "ssl"
+ :display-name "Use a secure connection (SSL)?"
+ :type :boolean
+ :default false}]
+ :sql-string-length-fn :CHAR_LENGTH
+ :column->base-type column->base-type
+ :connection-details->spec connection-details->spec
+ :unix-timestamp->timestamp unix-timestamp->timestamp
+ :date date
+ :date-interval date-interval
+ :timezone->set-timezone-sql timezone->set-timezone-sql
+ :driver-specific-sync-field! driver-specific-sync-field!
+ :humanize-connection-error-message humanize-connection-error-message}
+ sql-driver
+ (update :features conj :set-timezone)))
diff --git a/src/metabase/driver/query_processor.clj b/src/metabase/driver/query_processor.clj
index aa4863ffd0faf95159752a222e75b1e5c588cc66..f76f677925cd11cd6cb1c23ec33b66b12df5a750 100644
--- a/src/metabase/driver/query_processor.clj
+++ b/src/metabase/driver/query_processor.clj
@@ -8,7 +8,6 @@
[medley.core :as m]
[swiss.arrows :refer [<<-]]
[metabase.db :refer :all]
- [metabase.driver.interface :as i]
(metabase.driver.query-processor [annotate :as annotate]
[expand :as expand]
[interface :refer :all]
@@ -193,10 +192,10 @@
(fn [f]
(if-not (map? f) f
(m/filter-vals identity (into {} f))))
- ;; obscure DB details when logging. Just log the class of driver because we don't care about its properties
+ ;; obscure DB details when logging. Just log the name of driver because we don't care about its properties
(-> query
(assoc-in [:database :details] "😋 ") ; :yum:
- (update :driver class)))))))
+ (update :driver :driver-name)))))))
(qp query)))
@@ -241,8 +240,9 @@
(qp query))))
(defn- process-structured [{:keys [driver], :as query}]
- (let [driver-process-query (partial i/process-query driver)
- driver-wrap-process-query (partial i/wrap-process-query-middleware driver)]
+ (let [driver-process-query (:process-query driver)
+ driver-wrap-process-query (or (:process-query-in-context driver)
+ (fn [qp] qp))]
((<<- wrap-catch-exceptions
pre-expand
driver-wrap-process-query
@@ -257,8 +257,9 @@
driver-process-query) query)))
(defn- process-native [{:keys [driver], :as query}]
- (let [driver-process-query (partial i/process-query driver)
- driver-wrap-process-query (partial i/wrap-process-query-middleware driver)]
+ (let [driver-process-query (:process-query driver)
+ driver-wrap-process-query (or (:process-query-in-context driver)
+ (fn [qp] qp))]
((<<- wrap-catch-exceptions
driver-wrap-process-query
post-add-row-count-and-status
diff --git a/src/metabase/driver/query_processor/expand.clj b/src/metabase/driver/query_processor/expand.clj
index aee9f083c9d7c3c7652f14b9407cd606dd4ff7d4..fb1e3260e9073ffbde22fe9960bada756a101ace 100644
--- a/src/metabase/driver/query_processor/expand.clj
+++ b/src/metabase/driver/query_processor/expand.clj
@@ -24,15 +24,14 @@
;; ## -------------------- Expansion - Impl --------------------
-
-
(def ^:private ^:dynamic *original-query-dict*
"The entire original Query dict being expanded."
nil)
(defn- assert-driver-supports [^Keyword feature]
{:pre [(:driver *original-query-dict*)]}
- (driver/assert-driver-supports (:driver *original-query-dict*) feature))
+ (when-not (contains? (:features (:driver *original-query-dict*)) feature)
+ (throw (Exception. (format "%s is not supported by this driver." (name feature))))))
(defn- non-empty-clause? [clause]
(and clause
diff --git a/src/metabase/driver/sync.clj b/src/metabase/driver/sync.clj
index 624faf49cbc4cce0f14d20a338629eb2c7113e22..d116759e28009e759c5b12535b8ee2a2232d6c2d 100644
--- a/src/metabase/driver/sync.clj
+++ b/src/metabase/driver/sync.clj
@@ -8,7 +8,7 @@
[korma.core :as k]
[medley.core :as m]
[metabase.db :refer :all]
- (metabase.driver [interface :refer :all]
+ (metabase.driver [interface :refer [max-sync-lazy-seq-results]]
[query-processor :as qp])
[metabase.driver.sync.queries :as queries]
[metabase.events :as events]
@@ -38,56 +38,60 @@
;; ## sync-database! and sync-table!
+(defn -sync-database! [{:keys [active-table-names], :as driver} database]
+ (let [start-time (System/currentTimeMillis)
+ tracking-hash (str (java.util.UUID/randomUUID))]
+ (log/info (u/format-color 'magenta "Syncing %s database '%s'..." (name (:engine database)) (:name database)))
+ (events/publish-event :database-sync-begin {:database_id (:id database) :custom_id tracking-hash})
+
+ (let [active-table-names (active-table-names database)
+ table-name->id (sel :many :field->id [Table :name] :db_id (:id database) :active true)]
+ (assert (set? active-table-names) "active-table-names should return a set.")
+ (assert (every? string? active-table-names) "active-table-names should return the names of Tables as *strings*.")
+
+ ;; First, let's mark any Tables that are no longer active as such.
+ ;; These are ones that exist in table-name->id but not in active-table-names.
+ (doseq [[table-name table-id] table-name->id]
+ (when-not (contains? active-table-names table-name)
+ (upd Table table-id :active false)
+ (log/info (u/format-color 'cyan "Marked table %s.%s as inactive." (:name database) table-name))
+
+ ;; We need to mark driver Table's Fields as inactive so we don't expose them in UI such as FK selector (etc.)
+ (k/update Field
+ (k/where {:table_id table-id})
+ (k/set-fields {:active false}))))
+
+ ;; Next, we'll create new Tables (ones that came back in active-table-names but *not* in table-name->id)
+ (let [existing-table-names (set (keys table-name->id))
+ new-table-names (set/difference active-table-names existing-table-names)]
+ (when (seq new-table-names)
+ (log/debug (u/format-color 'blue "Found new tables: %s" new-table-names))
+ (doseq [new-table-name new-table-names]
+ ;; If it's a _metabase_metadata table then we'll handle later once everything else has been synced
+ (when-not (= (s/lower-case new-table-name) "_metabase_metadata")
+ (ins Table :db_id (:id database), :active true, :name new-table-name))))))
+
+ ;; Now sync the active tables
+ (->> (sel :many Table :db_id (:id database) :active true)
+ (map #(assoc % :db (delay database))) ; replace default delays with ones that reuse database (and don't require a DB call)
+ (sync-database-active-tables! driver))
+
+ ;; Ok, now if we had a _metabase_metadata table from earlier we can go ahead and sync from it
+ (sync-metabase-metadata-table! driver database)
+
+ (events/publish-event :database-sync-end {:database_id (:id database) :custom_id tracking-hash :running_time (- (System/currentTimeMillis) start-time)})
+ (log/info (u/format-color 'magenta "Finished syncing %s database %s. (%d ms)" (name (:engine database)) (:name database)
+ (- (System/currentTimeMillis) start-time)))))
+
(defn sync-database!
"Sync DATABASE and all its Tables and Fields."
- [driver database]
+ [{:keys [sync-in-context], :as driver} database]
(binding [qp/*disable-qp-logging* true
*sel-disable-logging* true]
- (sync-in-context driver database
- (fn []
- (let [start-time (System/currentTimeMillis)
- tracking-hash (str (java.util.UUID/randomUUID))]
- (log/info (u/format-color 'magenta "Syncing %s database '%s'..." (name (:engine database)) (:name database)))
- (events/publish-event :database-sync-begin {:database_id (:id database) :custom_id tracking-hash})
-
- (let [active-table-names (active-table-names driver database)
- table-name->id (sel :many :field->id [Table :name] :db_id (:id database) :active true)]
- (assert (set? active-table-names) "active-table-names should return a set.")
- (assert (every? string? active-table-names) "active-table-names should return the names of Tables as *strings*.")
-
- ;; First, let's mark any Tables that are no longer active as such.
- ;; These are ones that exist in table-name->id but not in active-table-names.
- (doseq [[table-name table-id] table-name->id]
- (when-not (contains? active-table-names table-name)
- (upd Table table-id :active false)
- (log/info (u/format-color 'cyan "Marked table %s.%s as inactive." (:name database) table-name))
-
- ;; We need to mark driver Table's Fields as inactive so we don't expose them in UI such as FK selector (etc.)
- (k/update Field
- (k/where {:table_id table-id})
- (k/set-fields {:active false}))))
-
- ;; Next, we'll create new Tables (ones that came back in active-table-names but *not* in table-name->id)
- (let [existing-table-names (set (keys table-name->id))
- new-table-names (set/difference active-table-names existing-table-names)]
- (when (seq new-table-names)
- (log/debug (u/format-color 'blue "Found new tables: %s" new-table-names))
- (doseq [new-table-name new-table-names]
- ;; If it's a _metabase_metadata table then we'll handle later once everything else has been synced
- (when-not (= (s/lower-case new-table-name) "_metabase_metadata")
- (ins Table :db_id (:id database), :active true, :name new-table-name))))))
-
- ;; Now sync the active tables
- (->> (sel :many Table :db_id (:id database) :active true)
- (map #(assoc % :db (delay database))) ; replace default delays with ones that reuse database (and don't require a DB call)
- (sync-database-active-tables! driver))
-
- ;; Ok, now if we had a _metabase_metadata table from earlier we can go ahead and sync from it
- (sync-metabase-metadata-table! driver database)
-
- (events/publish-event :database-sync-end {:database_id (:id database) :custom_id tracking-hash :running_time (- (System/currentTimeMillis) start-time)})
- (log/info (u/format-color 'magenta "Finished syncing %s database %s. (%d ms)" (name (:engine database)) (:name database)
- (- (System/currentTimeMillis) start-time))))))))
+ (let [f (partial -sync-database! driver database)]
+ (if sync-in-context
+ (sync-in-context database f)
+ (f)))))
(defn- sync-metabase-metadata-table!
"Databases may include a table named `_metabase_metadata` (case-insentive) which includes descriptions or other metadata about the `Tables` and `Fields`
@@ -103,35 +107,38 @@
`keypath` is of the form `table-name.key` or `table-name.field-name.key`, where `key` is the name of some property of `Table` or `Field`.
- This functionality is currently only used by the Sample Dataset."
- [driver database]
- (doseq [table-name (active-table-names driver database)]
- (when (= (s/lower-case table-name) "_metabase_metadata")
- (doseq [{:keys [keypath value]} (table-rows-seq driver database table-name)]
- (let [[_ table-name field-name k] (re-matches #"^([^.]+)\.(?:([^.]+)\.)?([^.]+)$" keypath)]
- (try (when (not= 1 (if field-name
- (k/update Field
- (k/where {:name field-name, :table_id (k/subselect Table
- (k/fields :id)
- (k/where {:db_id (:id database), :name table-name}))})
- (k/set-fields {(keyword k) value}))
- (k/update Table
- (k/where {:name table-name, :db_id (:id database)})
- (k/set-fields {(keyword k) value}))))
- (log/error (u/format-color "Error syncing _metabase_metadata: no matching keypath: %s" keypath)))
- (catch Throwable e
- (log/error (u/format-color 'red "Error in _metabase_metadata: %s" (.getMessage e))))))))))
+ This functionality is currently only used by the Sample Dataset. In order to use this functionality, drivers must implement optional fn `:table-rows-seq`."
+ [{:keys [table-rows-seq active-table-names]} database]
+ (when table-rows-seq
+ (doseq [table-name (active-table-names database)]
+ (when (= (s/lower-case table-name) "_metabase_metadata")
+ (doseq [{:keys [keypath value]} (table-rows-seq database table-name)]
+ (let [[_ table-name field-name k] (re-matches #"^([^.]+)\.(?:([^.]+)\.)?([^.]+)$" keypath)]
+ (try (when (not= 1 (if field-name
+ (k/update Field
+ (k/where {:name field-name, :table_id (k/subselect Table
+ (k/fields :id)
+ (k/where {:db_id (:id database), :name table-name}))})
+ (k/set-fields {(keyword k) value}))
+ (k/update Table
+ (k/where {:name table-name, :db_id (:id database)})
+ (k/set-fields {(keyword k) value}))))
+ (log/error (u/format-color "Error syncing _metabase_metadata: no matching keypath: %s" keypath)))
+ (catch Throwable e
+ (log/error (u/format-color 'red "Error in _metabase_metadata: %s" (.getMessage e)))))))))))
(defn sync-table!
"Sync a *single* TABLE by running all the sync steps for it.
This is used *instead* of `sync-database!` when syncing just one Table is desirable."
- [driver table]
- (let [database @(:db table)]
+ [{:keys [sync-in-context], :as driver} table]
+ (let [database @(:db table)
+ f (fn []
+ (sync-database-active-tables! driver [table])
+ (events/publish-event :table-sync {:table_id (:id table)}))]
(binding [qp/*disable-qp-logging* true]
- (sync-in-context driver database
- (fn []
- (sync-database-active-tables! driver [table])
- (events/publish-event :table-sync {:table_id (:id table)}))))))
+ (if sync-in-context
+ (sync-in-context database f)
+ (f)))))
;; ### sync-database-active-tables! -- runs the sync-table steps over sequence of Tables
@@ -237,10 +244,10 @@
(defn- sync-table-active-fields-and-pks!
"Create new Fields (and mark old ones as inactive) for TABLE, and update PK fields."
- [driver table]
+ [{:keys [active-column-names->type table-pks], :as driver} table]
(let [database @(:db table)]
;; Now do the syncing for Table's Fields
- (let [active-column-names->type (active-column-names->type driver table)
+ (let [active-column-names->type (active-column-names->type table)
existing-field-name->field (sel :many :field->fields [Field :name :base_type :id], :table_id (:id table), :active true, :parent_id nil)]
(assert (map? active-column-names->type) "active-column-names->type should return a map.")
@@ -274,7 +281,7 @@
;; TODO - we need to add functionality to update nested Field base types as well!
;; Now mark PK fields as such if needed
- (let [pk-fields (table-pks driver table)]
+ (let [pk-fields (table-pks table)]
(u/try-apply update-table-pks! table pk-fields)))))
@@ -292,9 +299,9 @@
(if (= field-count field-distinct-count) :1t1
:Mt1)))
-(defn- sync-table-fks! [driver table]
- (when (extends? ISyncDriverTableFKs (type driver))
- (let [fks (table-fks driver table)]
+(defn- sync-table-fks! [{:keys [features table-fks]} table]
+ (when (contains? features :foreign-keys)
+ (let [fks (table-fks table)]
(assert (and (set? fks)
(every? map? fks)
(every? :fk-column-name fks)
@@ -364,9 +371,9 @@
(defn- maybe-driver-specific-sync-field!
"If driver implements `ISyncDriverSpecificSyncField`, call `driver-specific-sync-field!`."
- [driver field]
- (when (satisfies? ISyncDriverSpecificSyncField driver)
- (driver-specific-sync-field! driver field)))
+ [{:keys [driver-specific-sync-field!]} field]
+ (when driver-specific-sync-field!
+ (driver-specific-sync-field! field)))
;; ### set-field-display-name-if-needed!
@@ -399,27 +406,29 @@
(inc non-nil-count)
more)))))
-(extend-protocol ISyncDriverFieldPercentUrls ; Default implementation
- Object
- (field-percent-urls [this field]
- (let [field-values (->> (field-values-lazy-seq this field)
- (filter identity)
- (take max-sync-lazy-seq-results))]
- (percent-valid-urls field-values))))
+(defn- default-field-percent-urls
+ "Default implementation for optional driver fn `:field-percent-urls` that calculates percentage in Clojure-land."
+ [{:keys [field-values-lazy-seq]} field]
+ (->> (field-values-lazy-seq field)
+ (filter identity)
+ (take max-sync-lazy-seq-results)
+ percent-valid-urls))
(defn- mark-url-field!
"If FIELD is texual, doesn't have a `special_type`, and its non-nil values are primarily URLs, mark it as `special_type` `url`."
- [driver field]
+ [{:keys [field-percent-urls], :as driver} field]
(when (and (not (:special_type field))
(contains? #{:CharField :TextField} (:base_type field)))
- (when-let [percent-urls (field-percent-urls driver field)]
- (assert (float? percent-urls))
- (assert (>= percent-urls 0.0))
- (assert (<= percent-urls 100.0))
- (when (> percent-urls percent-valid-url-threshold)
- (log/debug (u/format-color 'green "Field '%s' is %d%% URLs. Marking it as a URL." @(:qualified-name field) (int (math/round (* 100 percent-urls)))))
- (upd Field (:id field) :special_type :url)
- (assoc field :special_type :url)))))
+ (let [field-percent-urls (or field-percent-urls
+ (partial default-field-percent-urls driver))]
+ (when-let [percent-urls (field-percent-urls field)]
+ (assert (float? percent-urls))
+ (assert (>= percent-urls 0.0))
+ (assert (<= percent-urls 100.0))
+ (when (> percent-urls percent-valid-url-threshold)
+ (log/debug (u/format-color 'green "Field '%s' is %d%% URLs. Marking it as a URL." @(:qualified-name field) (int (math/round (* 100 percent-urls)))))
+ (upd Field (:id field) :special_type :url)
+ (assoc field :special_type :url))))))
;; ### mark-category-field-or-update-field-values!
@@ -455,26 +464,26 @@
"Fields whose values' average length is greater than this amount should be marked as `preview_display = false`."
50)
-(extend-protocol ISyncDriverFieldAvgLength ; Default implementation
- Object
- (field-avg-length [this field]
- (let [field-values (->> (field-values-lazy-seq this field)
- (filter identity)
- (take max-sync-lazy-seq-results)) ; as with field-percent-urls it's probably fine to consider the first 10,000 values rather than potentially millions
- field-values-count (count field-values)]
- (if (= field-values-count 0) 0
- (int (math/round (/ (->> field-values
- (map str)
- (map count)
- (reduce +))
- field-values-count)))))))
+(defn- default-field-avg-length [{:keys [field-values-lazy-seq]} field]
+ (let [field-values (->> (field-values-lazy-seq field)
+ (filter identity)
+ (take max-sync-lazy-seq-results))
+ field-values-count (count field-values)]
+ (if (= field-values-count 0) 0
+ (int (math/round (/ (->> field-values
+ (map str)
+ (map count)
+ (reduce +))
+ field-values-count))))))
(defn- mark-no-preview-display-field!
"If FIELD's is textual and its average length is too great, mark it so it isn't displayed in the UI."
- [driver field]
+ [{:keys [field-avg-length], :as driver} field]
(when (and (:preview_display field)
(contains? #{:CharField :TextField} (:base_type field)))
- (let [avg-len (field-avg-length driver field)]
+ (let [field-avg-length (or field-avg-length
+ (partial default-field-avg-length driver))
+ avg-len (field-avg-length field)]
(assert (integer? avg-len) "field-avg-length should return an integer.")
(when (> avg-len average-length-no-preview-threshold)
(log/debug (u/format-color 'green "Field '%s' has an average length of %d. Not displaying it in previews." @(:qualified-name field) avg-len))
@@ -506,10 +515,10 @@
(defn- mark-json-field!
"Mark FIELD as `:json` if it's textual, doesn't already have a special type, the majority of it's values are non-nil, and all of its non-nil values
are valid serialized JSON dictionaries or arrays."
- [driver field]
+ [{:keys [field-values-lazy-seq]} field]
(when (and (not (:special_type field))
(contains? #{:CharField :TextField} (:base_type field))
- (values-are-valid-json? (->> (field-values-lazy-seq driver field)
+ (values-are-valid-json? (->> (field-values-lazy-seq field)
(take max-sync-lazy-seq-results))))
(log/debug (u/format-color 'green "Field '%s' looks like it contains valid JSON objects. Setting special_type to :json." @(:qualified-name field)))
(upd Field (:id field) :special_type :json, :preview_display false)
@@ -593,11 +602,10 @@
(assoc field :special_type special-type))))
-(defn- sync-field-nested-fields! [driver field]
+(defn- sync-field-nested-fields! [{:keys [features active-nested-field-name->type], :as driver} field]
(when (and (= (:base_type field) :DictionaryField)
- (supports? driver :nested-fields) ; if one of these is true
- (satisfies? ISyncDriverFieldNestedFields driver)) ; the other should be :wink:
- (let [nested-field-name->type (active-nested-field-name->type driver field)]
+ (contains? features :nested-fields))
+ (let [nested-field-name->type (active-nested-field-name->type field)]
;; fetch existing nested fields
(let [existing-nested-field-name->id (sel :many :field->id [Field :name], :table_id (:table_id field), :active true, :parent_id (:id field))]
diff --git a/test/metabase/api/database_test.clj b/test/metabase/api/database_test.clj
index 9f7a6196a83d802f393b18458a42321431e32a6a..77c4c65ec0c73909bc7bcc3f36ebc39d08d8939b 100644
--- a/test/metabase/api/database_test.clj
+++ b/test/metabase/api/database_test.clj
@@ -25,7 +25,12 @@
;; ## GET /api/database/form_input
(expect
- {:engines driver/available-drivers
+ {:engines (into {} (for [[driver info] @driver/available-drivers]
+ {driver (-> info
+ (update :details-fields (partial map (fn [field]
+ (cond-> field
+ (:type field) (update :type name)))))
+ (update :features (partial map name)))}))
:timezones ["GMT"
"UTC"
"US/Alaska"
@@ -36,7 +41,7 @@
"US/Mountain"
"US/Pacific"
"America/Costa_Rica"]}
- ((user->client :crowberto) :get 200 "database/form_input"))
+ ((user->client :crowberto) :get 200 "database/form_input"))
;; # DB LIFECYCLE ENDPOINTS
diff --git a/test/metabase/driver/generic_sql_test.clj b/test/metabase/driver/generic_sql_test.clj
index f9026bc67a747701f1ad86928875e2a50d70533e..94a9a7d1bab9f5af578dabdc1296687251325cc9 100644
--- a/test/metabase/driver/generic_sql_test.clj
+++ b/test/metabase/driver/generic_sql_test.clj
@@ -1,9 +1,7 @@
(ns metabase.driver.generic-sql-test
(:require [expectations :refer :all]
[metabase.db :refer :all]
- [metabase.driver :as driver]
- (metabase.driver [h2 :as h2]
- [interface :as i])
+ [metabase.driver.h2 :refer [h2]]
[metabase.driver.generic-sql.util :refer [korma-entity]]
(metabase.models [field :refer [Field]]
[foreign-key :refer [ForeignKey]]
@@ -26,7 +24,7 @@
;; ACTIVE-TABLE-NAMES
(expect
#{"CATEGORIES" "VENUES" "CHECKINS" "USERS"}
- (i/active-table-names h2/driver (db)))
+ ((:active-table-names h2) (db)))
;; ACTIVE-COLUMN-NAMES->TYPE
(expect
@@ -36,15 +34,15 @@
"PRICE" :IntegerField
"CATEGORY_ID" :IntegerField
"ID" :BigIntegerField}
- (i/active-column-names->type h2/driver @venues-table))
+ ((:active-column-names->type h2) @venues-table))
;; ## TEST TABLE-PK-NAMES
;; Pretty straightforward
(expect #{"ID"}
- (i/table-pks h2/driver @venues-table))
+ ((:table-pks h2) @venues-table))
;; ## TEST FIELD-AVG-LENGTH
(expect 13
- (i/field-avg-length h2/driver @users-name-field))
+ ((:field-avg-length h2) @users-name-field))
diff --git a/test/metabase/driver/h2_test.clj b/test/metabase/driver/h2_test.clj
index 50b1d06cc6329693a224dc88220b3276ab376a53..013f6db7bb164cd14ef7df34045bbb8a96f5d926 100644
--- a/test/metabase/driver/h2_test.clj
+++ b/test/metabase/driver/h2_test.clj
@@ -1,19 +1,11 @@
(ns metabase.driver.h2-test
(:require [expectations :refer :all]
[metabase.db :as db]
- (metabase.driver [h2 :refer :all]
- [interface :refer [can-connect?]])
- [metabase.driver.generic-sql.interface :as i]
+ [metabase.driver.h2 :refer :all]
[metabase.test.util :refer [resolve-private-fns]]))
(resolve-private-fns metabase.driver.h2 connection-string->file+options file+options->connection-string connection-string-set-safe-options)
-;; # Check that database->connection-details works
-(expect {:db
- "file:/Users/cam/birdly/bird_sightings.db;AUTO_SERVER=TRUE;DB_CLOSE_DELAY=-1"}
- (i/database->connection-details driver {:details {:db "file:/Users/cam/birdly/bird_sightings.db;AUTO_SERVER=TRUE;DB_CLOSE_DELAY=-1"}}))
-
-
;; Check that the functions for exploding a connection string's options work as expected
(expect
["file:my-file" {"OPTION_1" "TRUE", "OPTION_2" "100", "LOOK_I_INCLUDED_AN_EXTRA_SEMICOLON" "NICE_TRY"}]
@@ -34,8 +26,7 @@
;; Make sure we *cannot* connect to a non-existent database
(expect :exception-thrown
- (try (can-connect? driver {:engine :h2
- :details {:db (str (System/getProperty "user.dir") "/toucan_sightings")}})
+ (try ((:can-connect? h2) {:db (str (System/getProperty "user.dir") "/toucan_sightings")})
(catch org.h2.jdbc.JdbcSQLException e
(and (re-matches #"Database .+ not found .+" (.getMessage e))
:exception-thrown))))
@@ -43,5 +34,4 @@
;; Check that we can connect to a non-existent Database when we enable potentailly unsafe connections (e.g. to the Metabase database)
(expect true
(binding [db/*allow-potentailly-unsafe-connections* true]
- (can-connect? driver {:engine :h2
- :details {:db (str (System/getProperty "user.dir") "/pigeon_sightings")}})))
+ ((:can-connect? h2) {:db (str (System/getProperty "user.dir") "/pigeon_sightings")})))
diff --git a/test/metabase/driver/mongo_test.clj b/test/metabase/driver/mongo_test.clj
index 7f460990eea55f8f5922fc8cc993bb38f1ff9a59..83f76b0441fd5aa9b83541f5e4430a2abfaf1e8c 100644
--- a/test/metabase/driver/mongo_test.clj
+++ b/test/metabase/driver/mongo_test.clj
@@ -4,8 +4,7 @@
[korma.core :as k]
[metabase.db :refer :all]
[metabase.driver :as driver]
- (metabase.driver [interface :as i]
- [mongo :as mongo])
+ [metabase.driver.mongo :refer [mongo]]
[metabase.driver.mongo.test-data :refer :all]
(metabase.models [field :refer [Field]]
[table :refer [Table]])
@@ -51,7 +50,7 @@
"Return an object that can be passed like a `Table` to driver sync functions."
[table-name]
{:pre [(keyword? table-name)]}
- {:db mongo-test-db
+ {:db mongo-test-db
:name (name table-name)})
(defn- field-name->fake-field
@@ -113,7 +112,7 @@
;; ### active-table-names
(expect-when-testing-mongo
#{"checkins" "categories" "users" "venues"}
- (i/active-table-names mongo/driver @mongo-test-db))
+ ((:active-table-names mongo) @mongo-test-db))
;; ### table->column-names
(expect-when-testing-mongo
@@ -154,14 +153,14 @@
{"_id" :IntegerField, "name" :TextField, "longitude" :FloatField, "latitude" :FloatField, "price" :IntegerField, "category_id" :IntegerField}] ; venues
(->> table-names
(map table-name->fake-table)
- (mapv (partial i/active-column-names->type mongo/driver))))
+ (mapv (:active-column-names->type mongo))))
;; ### table-pks
(expect-when-testing-mongo
[#{"_id"} #{"_id"} #{"_id"} #{"_id"}] ; _id for every table
(->> table-names
(map table-name->fake-table)
- (mapv (partial i/table-pks mongo/driver))))
+ (mapv (:table-pks mongo))))
;; ## Big-picture tests for the way data should look post-sync
diff --git a/test/metabase/driver/postgres_test.clj b/test/metabase/driver/postgres_test.clj
index 1f595f25e907edb865935d004e2e956f2d1452c6..7b2e3ff94b587648c253c7efbfac4cdc70c3a4f7 100644
--- a/test/metabase/driver/postgres_test.clj
+++ b/test/metabase/driver/postgres_test.clj
@@ -1,25 +1,10 @@
(ns metabase.driver.postgres-test
(:require [expectations :refer :all]
- [metabase.driver.generic-sql.interface :as i]
- [metabase.driver.postgres :refer :all]
+ [metabase.driver.postgres :refer [postgres]]
(metabase.test.data [datasets :refer [expect-with-dataset]]
[interface :refer [def-database-definition]])
[metabase.test.util.q :refer [Q]]))
-;; # Check that database->connection details still works whether we're dealing with new-style or legacy details
-;; ## new-style
-(expect {:db "bird_sightings"
- :ssl false
- :port 5432
- :host "localhost"
- :user "camsaul"}
- (i/database->connection-details driver {:details {:ssl false
- :host "localhost"
- :port 5432
- :dbname "bird_sightings"
- :user "camsaul"}}))
-
-
;; # Check that SSL params get added the connection details in the way we'd like
;; ## no SSL -- this should *not* include the key :ssl (regardless of its value) since that will cause the PG driver to use SSL anyway
(expect
@@ -28,11 +13,11 @@
:subprotocol "postgresql"
:subname "//localhost:5432/bird_sightings"
:make-pool? true}
- (i/connection-details->connection-spec driver {:ssl false
- :host "localhost"
- :port 5432
- :dbname "bird_sightings"
- :user "camsaul"}))
+ ((:connection-details->spec postgres) {:ssl false
+ :host "localhost"
+ :port 5432
+ :dbname "bird_sightings"
+ :user "camsaul"}))
;; ## ssl - check that expected params get added
(expect
@@ -44,11 +29,11 @@
:user "camsaul"
:sslfactory "org.postgresql.ssl.NonValidatingFactory"
:subname "//localhost:5432/bird_sightings"}
- (i/connection-details->connection-spec driver {:ssl true
- :host "localhost"
- :port 5432
- :dbname "bird_sightings"
- :user "camsaul"}))
+ ((:connection-details->spec postgres) {:ssl true
+ :host "localhost"
+ :port 5432
+ :dbname "bird_sightings"
+ :user "camsaul"}))
;;; # UUID Support
(def-database-definition ^:private with-uuid
["users"