Skip to content
Snippets Groups Projects
Commit 3d9b139b authored by Cam Saül's avatar Cam Saül
Browse files

Rework the way drivers are defined

parent 667d5735
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Showing
with 679 additions and 585 deletions
src/metabase/api/database.clj
+ 2
− 2
View file @ 3d9b139b
......@@ -17,7 +17,7 @@
(defannotation DBEngine
"Param must be a valid database engine type, e.g. `h2` or `postgres`."
[symb value :nillable]
(checkp-contains? (set (map name (keys driver/available-drivers))) symb value))
(checkp-contains? (set (map name (keys @driver/available-drivers))) symb value))
(defendpoint GET "/"
"Fetch all `Databases`."
......@@ -46,7 +46,7 @@
"Values of options for the create/edit `Database` UI."
[]
{:timezones metabase.models.common/timezones
:engines driver/available-drivers})
:engines @driver/available-drivers})
;; Stub function that will eventually validate a connection string
(defendpoint POST "/validate"
......
src/metabase/api/setup.clj
+ 2
− 6
View file @ 3d9b139b
(ns metabase.api.setup
(:require [compojure.core :refer [defroutes POST]]
[metabase.api.common :refer :all]
(metabase.api [common :refer :all]
[database :refer [annotation:DBEngine]])
[metabase.db :refer :all]
[metabase.driver :as driver]
[metabase.events :as events]
......@@ -11,11 +12,6 @@
[metabase.setup :as setup]
[metabase.util :as u]))
(defannotation DBEngine
"Param must be a valid database engine type, e.g. `h2` or `postgres`."
[symb value :nillable]
(checkp-contains? (set (map name (keys driver/available-drivers))) symb value))
(defannotation SetupToken
"Check that param matches setup token or throw a 403."
[symb value]
......
src/metabase/driver.clj
+ 34
− 31
View file @ 3d9b139b
......@@ -2,10 +2,10 @@
(:require clojure.java.classpath
[clojure.string :as s]
[clojure.tools.logging :as log]
[clojure.tools.namespace.find :as ns-find]
[medley.core :as m]
[metabase.db :refer [ins sel upd]]
(metabase.driver [interface :as i]
[query-processor :as qp])
[metabase.driver.query-processor :as qp]
(metabase.models [database :refer [Database]]
[query-execution :refer [QueryExecution]])
[metabase.models.setting :refer [defsetting]]
......@@ -13,30 +13,32 @@
(declare -dataset-query query-fail query-complete save-query-execution)
;; ## CONFIG
;;; ## CONFIG
(defsetting report-timezone "Connection timezone to use when executing queries. Defaults to system timezone.")
;; ## Constants
(def ^:const available-drivers
"Available DB drivers."
{:h2 {:id "h2"
:name "H2"}
:postgres {:id "postgres"
:name "Postgres"}
:mongo {:id "mongo"
:name "MongoDB"}
:mysql {:id "mysql"
:name "MySQL"}})
(def available-drivers
"Delay to a map of info about available drivers."
(delay (->> (for [namespace (->> (ns-find/find-namespaces (clojure.java.classpath/classpath))
(filter (fn [ns-symb]
(re-matches #"^metabase\.driver\.[a-z0-9_]+$" (name ns-symb)))))]
(do (require namespace)
(->> (ns-publics namespace)
(map (fn [[symb varr]]
(when (::driver (meta varr))
{(keyword symb) (select-keys @varr [:details-fields
:driver-name
:features])})))
(into {}))))
(into {}))))
(defn is-engine?
"Predicate function which validates if the given argument represents a valid driver identifier."
"Is ENGINE a valid driver name?"
[engine]
(if (not (nil? engine))
(contains? (set (map name (keys available-drivers))) (name engine))
false))
(when engine
(contains? (set (keys @available-drivers)) (keyword engine))))
(defn class->base-type
"Return the `Field.base_type` that corresponds to a given class returned by the DB."
......@@ -65,13 +67,11 @@
"Return the driver instance that should be used for given ENGINE.
This loads the corresponding driver if needed; it is expected that it resides in a var named
metabase.driver.<engine>/driver"
metabase.driver.<engine>/<engine>"
[engine]
{:pre [(keyword? engine)
(contains? (set (keys available-drivers)) engine)]}
(let [nmspc (symbol (format "metabase.driver.%s" (name engine)))]
(require nmspc)
@(ns-resolve nmspc 'driver)))
@(ns-resolve nmspc (symbol (name engine)))))
;; Can the type of a DB change?
......@@ -93,11 +93,12 @@
"Check whether we can connect to DATABASE and perform a basic query (such as `SELECT 1`)."
[database]
{:pre [(map? database)]}
(try
(i/can-connect? (engine->driver (:engine database)) database)
(catch Throwable e
(log/error "Failed to connect to database:" (.getMessage e))
false)))
(let [driver (engine->driver (:engine database))]
(try
((:can-connect? driver) (:details database))
(catch Throwable e
(log/error "Failed to connect to database:" (.getMessage e))
false))))
(defn can-connect-with-details?
"Check whether we can connect to a database with ENGINE and DETAILS-MAP and perform a basic query.
......@@ -107,15 +108,17 @@
(can-connect-with-details? :postgres {:host \"localhost\", :port 5432, ...})"
[engine details-map & [rethrow-exceptions]]
{:pre [(keyword? engine)
(contains? (set (keys available-drivers)) engine)
(is-engine? engine)
(map? details-map)]}
(let [driver (engine->driver engine)]
(let [{:keys [can-connect? humanize-connection-error-message]} (engine->driver engine)]
(try
(i/can-connect-with-details? driver details-map)
(can-connect? details-map)
(catch Throwable e
(log/error "Failed to connect to database:" (.getMessage e))
(when rethrow-exceptions
(let [message (i/humanize-connection-error-message driver (.getMessage e))]
(let [^String message ((or humanize-connection-error-message
identity)
(.getMessage e))]
(throw (Exception. message))))
false))))
......
src/metabase/driver/generic_sql.clj
+ 100
− 57
View file @ 3d9b139b
......@@ -4,19 +4,13 @@
[korma.core :as k]
[korma.sql.utils :as utils]
[metabase.driver :as driver]
(metabase.driver [interface :refer [max-sync-lazy-seq-results IDriver ISyncDriverTableFKs ISyncDriverFieldAvgLength ISyncDriverFieldPercentUrls]]
(metabase.driver [interface :refer [max-sync-lazy-seq-results defdriver]]
[sync :as driver-sync])
(metabase.driver.generic-sql [interface :as i]
[query-processor :as qp]
(metabase.driver.generic-sql [query-processor :as qp]
[util :refer :all])
[metabase.models.field :as field]
[metabase.util :as u]))
(def ^:const features
"Features supported by *all* Generic SQL drivers."
#{:foreign-keys
:standard-deviation-aggregations
:unix-timestamp-special-type-fields})
(def ^:private ^:const field-values-lazy-seq-chunk-size
"How many Field values should we fetch at a time for `field-values-lazy-seq`?"
;; Hopefully this is a good balance between
......@@ -25,35 +19,28 @@
;; 3. Not fetching too many results for things like mark-json-field! which will fail after the first result that isn't valid JSON
500)
(defn- can-connect-with-details? [driver details]
(let [connection (i/connection-details->connection-spec driver details)]
(defn- can-connect? [connection-details->spec details]
(let [connection (connection-details->spec details)]
(= 1 (-> (k/exec-raw connection "SELECT 1" :results)
first
vals
first))))
(defn- can-connect? [driver database]
(can-connect-with-details? driver (i/database->connection-details driver database)))
(defn- wrap-process-query-middleware [_ qp]
(fn [query]
(qp query)))
(defn- process-query [_ query]
(defn- process-query [query]
(qp/process-and-run query))
(defn- sync-in-context [_ database do-sync-fn]
(defn- sync-in-context [database do-sync-fn]
(with-jdbc-metadata [_ database]
(do-sync-fn)))
(defn- active-table-names [_ database]
(defn- active-table-names [database]
(with-jdbc-metadata [^java.sql.DatabaseMetaData md database]
(->> (.getTables md nil nil nil (into-array String ["TABLE", "VIEW"]))
jdbc/result-set-seq
(map :table_name)
set)))
(defn- active-column-names->type [{:keys [column->base-type]} table]
(defn- active-column-names->type [column->base-type table]
{:pre [(map? column->base-type)]}
(with-jdbc-metadata [^java.sql.DatabaseMetaData md @(:db table)]
(->> (.getColumns md nil nil (:name table) nil)
......@@ -64,14 +51,14 @@
:UnknownField)}))
(into {}))))
(defn- table-pks [_ table]
(defn- table-pks [table]
(with-jdbc-metadata [^java.sql.DatabaseMetaData md @(:db table)]
(->> (.getPrimaryKeys md nil nil (:name table))
jdbc/result-set-seq
(map :column_name)
set)))
(defn- field-values-lazy-seq [_ {:keys [qualified-name-components table], :as field}]
(defn- field-values-lazy-seq [{:keys [qualified-name-components table], :as field}]
(assert (and (map? field)
(delay? qualified-name-components)
(delay? table))
......@@ -96,12 +83,12 @@
(fetch-chunk 0 field-values-lazy-seq-chunk-size
max-sync-lazy-seq-results)))
(defn- table-rows-seq [_ database table-name]
(defn- table-rows-seq [database table-name]
(k/select (-> (k/create-entity table-name)
(k/database (db->korma-db database)))))
(defn- table-fks [_ table]
(defn- table-fks [table]
(with-jdbc-metadata [^java.sql.DatabaseMetaData md @(:db table)]
(->> (.getImportedKeys md nil nil (:name table))
jdbc/result-set-seq
......@@ -111,8 +98,7 @@
:dest-column-name (:pkcolumn_name result)}))
set)))
(defn- field-avg-length [{:keys [sql-string-length-fn], :as driver} field]
{:pre [(keyword? sql-string-length-fn)]}
(defn- field-avg-length [sql-string-length-fn field]
(or (some-> (korma-entity @(:table field))
(k/select (k/aggregate (avg (k/sqlfn* sql-string-length-fn
(utils/func "CAST(%s AS CHAR)"
......@@ -123,7 +109,7 @@
int)
0))
(defn- field-percent-urls [_ field]
(defn- field-percent-urls [field]
(or (let [korma-table (korma-entity @(:table field))]
(when-let [total-non-null-count (:count (first (k/select korma-table
(k/aggregate (count :*) :count)
......@@ -135,31 +121,88 @@
(float (/ url-count total-non-null-count))))))
0.0))
(def ^:const GenericSQLIDriverMixin
"Generic SQL implementation of the `IDriver` protocol.
(extend H2Driver
IDriver
GenericSQLIDriverMixin)"
{:can-connect? can-connect?
:can-connect-with-details? can-connect-with-details?
:wrap-process-query-middleware wrap-process-query-middleware
:process-query process-query
:sync-in-context sync-in-context
:active-table-names active-table-names
:active-column-names->type active-column-names->type
:table-pks table-pks
:field-values-lazy-seq field-values-lazy-seq
:table-rows-seq table-rows-seq})
(def ^:const GenericSQLISyncDriverTableFKsMixin
"Generic SQL implementation of the `ISyncDriverTableFKs` protocol."
{:table-fks table-fks})
(def ^:const GenericSQLISyncDriverFieldAvgLengthMixin
"Generic SQL implementation of the `ISyncDriverFieldAvgLengthMixin` protocol."
{:field-avg-length field-avg-length})
(def ^:const GenericSQLISyncDriverFieldPercentUrlsMixin
"Generic SQL implementation of the `ISyncDriverFieldPercentUrls` protocol."
{:field-percent-urls field-percent-urls})
(def ^:private ^:const required-fns
"Functions that concrete SQL drivers must define."
#{:connection-details->spec
:unix-timestamp->timestamp
:date
:date-interval})
(defn- verify-sql-driver [{:keys [column->base-type sql-string-length-fn], :as driver}]
;; Check the :column->base-type map
(assert column->base-type
"SQL drivers must define :column->base-type.")
(assert (map? column->base-type)
":column->base-type should be a map")
(doseq [[k v] column->base-type]
(assert (keyword? k)
(format "Not a keyword: %s" k))
(assert (contains? field/base-types v)
(format "Invalid field base-type: %s" v)))
;; Check :sql-string-length-fn
(assert sql-string-length-fn
"SQL drivers must define :sql-string-length-fn.")
(assert (keyword? sql-string-length-fn)
":sql-string-length-fn must be a keyword.")
;; Check required fns
(doseq [f required-fns]
(assert (f driver)
(format "SQL drivers must define %s." f))
(assert (fn? (f driver))
(format "%s must be a fn." f))))
(defn sql-driver
"Create a Metabase DB driver using the Generic SQL functions.
A SQL driver must define the following properties / functions:
* `column->base-type`
A map of native DB column types (as keywords) to the `Field` `base-types` they map to.
* `sql-string-length-fn`
Keyword name of the SQL function that should be used to get the length of a string, e.g. `:LENGTH`.
* `(connection-details->spec [details-map])`
Given a `Database` DETAILS-MAP, return a JDBC connection spec.
* `(unix-timestamp->timestamp [seconds-or-milliseconds field-or-value])`
Return a korma form appropriate for converting a Unix timestamp integer field or value to an proper SQL `Timestamp`.
SECONDS-OR-MILLISECONDS refers to the resolution of the int in question and with be either `:seconds` or `:milliseconds`.
* `(timezone->set-timezone-sql [timezone])`
Return a string that represents the SQL statement that should be used to set the timezone
for the current transaction.
* `(date [this ^Keyword unit field-or-value])`
Return a korma form for truncating a date or timestamp field or value to a given resolution, or extracting a
date component.
* `(date-interval [unit amount])`
Return a korma form for a date relative to NOW(), e.g. on that would produce SQL like `(NOW() + INTERVAL '1 month')`."
[driver]
;; Verify the driver
(verify-sql-driver driver)
(merge
{:features #{:foreign-keys
:standard-deviation-aggregations
:unix-timestamp-special-type-fields}
:can-connect? (partial can-connect? (:connection-details->spec driver))
:process-query process-query
:sync-in-context sync-in-context
:active-table-names active-table-names
:active-column-names->type (partial active-column-names->type (:column->base-type driver))
:table-pks table-pks
:field-values-lazy-seq field-values-lazy-seq
:table-rows-seq table-rows-seq
:table-fks table-fks
:field-avg-length (partial field-avg-length (:sql-string-length-fn driver))}
driver))
src/metabase/driver/generic_sql/interface.clj deleted 100644 → 0
+ 0
− 26
View file @ 667d5735
(ns metabase.driver.generic-sql.interface
(:import clojure.lang.Keyword))
(defprotocol ISqlDriverDatabaseSpecific
"Methods a DB-specific concrete SQL driver should implement.
They should also have the following properties:
* `column->base-type`
* `sql-string-length-fn`"
(connection-details->connection-spec [this connection-details])
(database->connection-details [this database])
(unix-timestamp->timestamp [this ^Keyword seconds-or-milliseconds field-or-value]
"Return a korma form appropriate for converting a Unix timestamp integer field or value to an proper SQL `Timestamp`.
SECONDS-OR-MILLISECONDS refers to the resolution of the int in question and with be either `:seconds` or `:milliseconds`.")
(timezone->set-timezone-sql [this timezone]
"Return a string that represents the SQL statement that should be used to set the timezone
for the current transaction.")
(date [this ^Keyword unit field-or-value]
"Return a korma form for truncating a date or timestamp field or value to a given resolution, or extracting a date component.")
(date-interval [this ^Keyword unit ^Integer amount]
"Return a korma form for a date relative to NOW(), e.g. on that would produce SQL like `(NOW() + INTERVAL '1 month')`."))
src/metabase/driver/generic_sql/native.clj
+ 4
− 6
View file @ 3d9b139b
......@@ -6,9 +6,7 @@
db)
[metabase.db :refer [sel]]
[metabase.driver :as driver]
[metabase.driver.interface :refer [supports?]]
(metabase.driver.generic-sql [interface :as i]
[util :refer :all])
[metabase.driver.generic-sql.util :refer :all]
[metabase.models.database :refer [Database]]))
(defn- value->base-type
......@@ -34,10 +32,10 @@
(when-let [timezone (or (-> query :native :timezone)
(driver/report-timezone))]
(when (seq timezone)
(let [driver (driver/engine->driver (:engine database))]
(when (supports? driver :set-timezone)
(let [{:keys [features timezone->set-timezone-sql]} (driver/engine->driver (:engine database))]
(when (contains? features :set-timezone)
(log/debug "Setting timezone to:" timezone)
(jdbc/db-do-prepared conn (i/timezone->set-timezone-sql driver timezone))))))
(jdbc/db-do-prepared conn (timezone->set-timezone-sql timezone))))))
(jdbc/query conn sql :as-arrays? true))]
;; TODO - Why don't we just use annotate?
{:rows rows
......
src/metabase/driver/generic_sql/query_processor.clj
+ 11
− 13
View file @ 3d9b139b
......@@ -8,10 +8,8 @@
[korma.sql.utils :as utils]
[metabase.config :as config]
[metabase.driver :as driver]
(metabase.driver [interface :refer [supports?]]
[query-processor :as qp])
(metabase.driver.generic-sql [interface :as i]
[native :as native]
[metabase.driver.query-processor :as qp]
(metabase.driver.generic-sql [native :as native]
[util :refer :all])
[metabase.util :as u])
(:import java.sql.Timestamp
......@@ -42,9 +40,9 @@
(mapcat #(if (vector? %) % [%]))))
set-timezone-sql (when-let [timezone (driver/report-timezone)]
(when (seq timezone)
(let [driver (:driver *query*)]
(when (supports? driver :set-timezone)
`(exec-raw ~(i/timezone->set-timezone-sql driver timezone))))))
(let [{:keys [features timezone->set-timezone-sql]} (:driver *query*)]
(when (contains? features :set-timezone)
`(exec-raw ~(timezone->set-timezone-sql timezone))))))
korma-form `(let [~'entity (korma-entity ~database ~source-table)]
~(if set-timezone-sql `(korma.db/with-db (:db ~'entity)
(korma.db/transaction
......@@ -100,7 +98,7 @@
([this]
(formatted this false))
([{:keys [table-name special-type field-name], :as field} include-as?]
(let [->timestamp (partial i/unix-timestamp->timestamp (:driver *query*))
(let [->timestamp (:unix-timestamp->timestamp (:driver *query*))
field (cond-> (keyword (str table-name \. field-name))
(= special-type :timestamp_seconds) (->timestamp :seconds)
(= special-type :timestamp_milliseconds) (->timestamp :milliseconds))]
......@@ -112,7 +110,7 @@
([this]
(formatted this false))
([{unit :unit, {:keys [field-name base-type special-type], :as field} :field} include-as?]
(let [field (i/date (:driver *query*) unit (formatted field))]
(let [field ((:date (:driver *query*)) unit (formatted field))]
(if include-as? [field (keyword field-name)]
field))))
......@@ -145,7 +143,7 @@
(formatted this false))
([{value :value, {unit :unit} :field} _]
;; prevent Clojure from converting this to #inst literal, which is a util.date
(i/date (:driver *query*) unit `(Timestamp/valueOf ~(.toString value)))))
((:date (:driver *query*)) unit `(Timestamp/valueOf ~(.toString value)))))
RelativeDateTimeValue
(formatted
......@@ -153,9 +151,9 @@
(formatted this false))
([{:keys [amount unit], {field-unit :unit} :field} _]
(let [driver (:driver *query*)]
(i/date driver field-unit (if (zero? amount)
(sqlfn :NOW)
(i/date-interval driver unit amount)))))))
((:date driver) field-unit (if (zero? amount)
(sqlfn :NOW)
((:date-interval driver) unit amount)))))))
(defmethod apply-form :aggregation [[_ {:keys [aggregation-type field]}]]
......
src/metabase/driver/generic_sql/util.clj
+ 4
− 7
View file @ 3d9b139b
......@@ -7,19 +7,16 @@
[db :as kdb])
[korma.sql.utils :as utils]
[metabase.driver :as driver]
[metabase.driver.query-processor :as qp]
[metabase.driver.generic-sql.interface :as i]))
[metabase.driver.query-processor :as qp]))
(defn- db->connection-spec
"Return a JDBC connection spec for a Metabase `Database`."
[{{:keys [short-lived?]} :details, :as database}]
(let [driver (driver/engine->driver (:engine database))
database->connection-details (partial i/database->connection-details driver)
connection-details->connection-spec (partial i/connection-details->connection-spec driver)]
(merge (-> database database->connection-details connection-details->connection-spec)
(let [{:keys [connection-details->spec]} (driver/engine->driver (:engine database))]
(assoc (connection-details->spec (:details database))
;; unless this is a temp DB, we need to make a pool or the connection will be closed before we get a chance to unCLOB-er the results during JSON serialization
;; TODO - what will we do once we have CLOBS in temp DBs?
{:make-pool? (not short-lived?)})))
:make-pool? (not short-lived?))))
(def ^{:arglists '([database])}
db->korma-db
......
src/metabase/driver/h2.clj
+ 23
− 36
View file @ 3d9b139b
......@@ -3,12 +3,9 @@
[korma.db :as kdb]
[korma.sql.utils :as utils]
[metabase.db :as db]
[metabase.driver :as driver]
(metabase.driver [generic-sql :as generic-sql, :refer [GenericSQLIDriverMixin GenericSQLISyncDriverTableFKsMixin
GenericSQLISyncDriverFieldAvgLengthMixin GenericSQLISyncDriverFieldPercentUrlsMixin]]
[interface :as i, :refer [IDriver ISyncDriverTableFKs ISyncDriverFieldAvgLength ISyncDriverFieldPercentUrls]])
(metabase.driver.generic-sql [interface :refer [ISqlDriverDatabaseSpecific]]
[util :refer [funcs]])
(metabase.driver [generic-sql :refer [sql-driver]]
[interface :as i, :refer [defdriver]])
[metabase.driver.generic-sql.util :refer [funcs]]
[metabase.models.database :refer [Database]]))
(def ^:private ^:const column->base-type
......@@ -104,20 +101,17 @@
(file+options->connection-string file (merge options {"IFEXISTS" "TRUE"
"ACCESS_MODE_DATA" "r"}))))
(defn- connection-details->connection-spec [_ details]
(defn- connection-details->spec [details]
(kdb/h2 (if db/*allow-potentailly-unsafe-connections* details
(update details :db connection-string-set-safe-options))))
(defn- database->connection-details [_ {:keys [details]}]
details)
(defn- unix-timestamp->timestamp [_ field-or-value seconds-or-milliseconds]
(defn- unix-timestamp->timestamp [field-or-value seconds-or-milliseconds]
(utils/func (format "TIMESTAMPADD('%s', %%s, TIMESTAMP '1970-01-01T00:00:00Z')" (case seconds-or-milliseconds
:seconds "SECOND"
:milliseconds "MILLISECOND"))
[field-or-value]))
(defn- wrap-process-query-middleware [_ qp]
(defn- process-query-in-context [qp]
(fn [{query-type :type, :as query}]
{:pre [query-type]}
;; For :native queries check to make sure the DB in question has a (non-default) NAME property specified in the connection string.
......@@ -153,7 +147,7 @@
["YEAR(%s)" field-or-value]
["((QUARTER(%s) * 3) - 2)" field-or-value]]))
(defn- date [_ unit field-or-value]
(defn- date [unit field-or-value]
(if (= unit :quarter)
(trunc-to-quarter field-or-value)
(utils/func (case unit
......@@ -175,7 +169,7 @@
[field-or-value])))
;; TODO - maybe rename this relative-date ?
(defn- date-interval [_ unit amount]
(defn- date-interval [unit amount]
(utils/generated (format (case unit
:minute "DATEADD('MINUTE', %d, NOW())"
:hour "DATEADD('HOUR', %d, NOW())"
......@@ -186,7 +180,7 @@
:year "DATEADD('YEAR', %d, NOW())")
amount)))
(defn- humanize-connection-error-message [_ message]
(defn- humanize-connection-error-message [message]
(condp re-matches message
#"^A file path that is implicitly relative to the current working directory is not allowed in the database URL .*$"
(i/connection-error-messages :cannot-connect-check-host-and-port)
......@@ -200,24 +194,17 @@
#".*" ; default
message))
(defrecord H2Driver [])
(extend H2Driver
ISqlDriverDatabaseSpecific {:connection-details->connection-spec connection-details->connection-spec
:database->connection-details database->connection-details
:date date
:date-interval date-interval
:unix-timestamp->timestamp unix-timestamp->timestamp}
;; Override the generic SQL implementation of wrap-process-query-middleware so we can block unsafe native queries (see above)
IDriver (assoc GenericSQLIDriverMixin
:humanize-connection-error-message humanize-connection-error-message
:wrap-process-query-middleware wrap-process-query-middleware)
ISyncDriverTableFKs GenericSQLISyncDriverTableFKsMixin
ISyncDriverFieldAvgLength GenericSQLISyncDriverFieldAvgLengthMixin
ISyncDriverFieldPercentUrls GenericSQLISyncDriverFieldPercentUrlsMixin)
(def ^:const driver
(map->H2Driver {:column->base-type column->base-type
:features generic-sql/features
:sql-string-length-fn :LENGTH}))
(defdriver h2
(sql-driver {:driver-name "H2"
:details-fields [{:name "db"
:display-name "Connection String"
:placeholder "file:/Users/camsaul/bird_sightings/toucans;AUTO_SERVER=TRUE"
:required true}]
:column->base-type column->base-type
:sql-string-length-fn :LENGTH
:connection-details->spec connection-details->spec
:date date
:date-interval date-interval
:unix-timestamp->timestamp unix-timestamp->timestamp
:humanize-connection-error-message humanize-connection-error-message
:process-query-in-context process-query-in-context}))
src/metabase/driver/interface.clj
+ 214
− 134
View file @ 3d9b139b
(ns metabase.driver.interface
"Protocols that DB drivers implement. Thus, the interface such drivers provide."
(:import (clojure.lang Keyword)))
(def ^:const driver-optional-features
"A set on optional features (as keywords) that may or may not be supported by individual drivers."
#{:foreign-keys
:nested-fields ; are nested Fields (i.e., Mongo-style nested keys) supported?
:set-timezone
:standard-deviation-aggregations
:unix-timestamp-special-type-fields})
(ns metabase.driver.interface)
(def ^:const max-sync-lazy-seq-results
"The maximum number of values we should return when using `field-values-lazy-seq`.
......@@ -26,135 +16,225 @@
:username-incorrect "Looks like your username is incorrect."
:username-or-password-incorrect "Looks like the username or password is incorrect."})
;; ## IDriver Protocol
(def ^:private ^:const feature->required-fns
"Map of optional driver features (as keywords) to a set of functions drivers that support that feature must define."
{:foreign-keys #{:table-fks}
:nested-fields #{:active-nested-field-name->type}
:set-timezone nil
:standard-deviation-aggregations nil
:unix-timestamp-special-type-fields nil})
(def ^:private ^:const optional-features
(set (keys feature->required-fns)))
(def ^:private ^:const required-fns
#{:can-connect?
:active-table-names
:active-column-names->type
:table-pks
:field-values-lazy-seq
:process-query})
(def ^:private ^:const optional-fns
#{:humanize-connection-error-message
:sync-in-context
:process-query-in-context
:table-rows-seq
:field-avg-length
:field-percent-urls
:driver-specific-sync-field!})
(defn verify-driver
"Verify that a Metabase DB driver contains the expected properties and that they are the correct type."
[{:keys [driver-name details-fields features], :as driver}]
;; Check :driver-name is a string
(assert driver-name
"Missing property :driver-name.")
(assert (string? driver-name)
":driver-name must be a string.")
;; Check the :details-fields
(assert details-fields
"Driver is missing property :details-fields.")
(assert (vector? details-fields)
":details-fields should be a vector.")
(doseq [f details-fields]
(assert (map? f)
(format "Details fields must be maps: %s" f))
(assert (:name f)
(format "Details field %s is missing a :name property." f))
(assert (:display-name f)
(format "Details field %s is missing a :display-name property." f))
(when (:type f)
(assert (contains? #{:string :integer :password :boolean} (:type f))
(format "Invalid type %s in details field %s." (:type f) f)))
(when (:default f)
(assert (not (:placeholder f))
(format "Fields should not define both :default and :placeholder: %s" f))
(assert (not (:required f))
(format "Fields that define a :default cannot be :required: %s" f))))
(defprotocol IDriver
"Methods all drivers must implement.
They should also include the following properties:
;; Check that all required functions are defined
(doseq [f required-fns]
(assert (f driver)
(format "Missing fn: %s" f))
(assert (fn? (f driver))
(format "Not a fn: %s" (f driver))))
* `features` (optional)
A set containing one or more `driver-optional-features`"
;; Check that all features declared are valid
(when features
(assert (and (set? features)
(every? keyword? features))
":features must be a set of keywords.")
(doseq [feature features]
(assert (contains? optional-features feature)
(format "Not a valid feature: %s" feature))
(doseq [f (feature->required-fns feature)]
(assert (f driver)
(format "Drivers that support feature %s must have fn %s." feature f))
(assert (fn? (f driver))
(format "Not a fn: %s" f)))))
;; Connection
(can-connect? [this database]
"Check whether we can connect to DATABASE and perform a simple query.
(To check whether we can connect to a database given only its details, use `can-connect-with-details?` instead).
;; Check that the optional fns, if included, are actually fns
(doseq [f optional-fns]
(when (f driver)
(assert (fn? (f driver))
(format "Not a fn: %s" f)))))
(can-connect? driver (sel :one Database :id 1))")
(can-connect-with-details? [this details-map]
"Check whether we can connect to a database and performa a simple query.
Returns true if we can, otherwise returns `false` or throws an `Exception`.
(defmacro defdriver
"Define and validate a new Metabase DB driver.
(can-connect-with-details? driver {:engine :postgres, :dbname \"book\", ...})")
All drivers must include the following keys:
(humanize-connection-error-message ^String [this ^String message]
"Return a humanized (user-facing) version of an connection error message string.
Generic error messages are provided in the constant `connection-error-messages`; return one of these whenever possible.")
#### PROPERTIES
;; Syncing
(sync-in-context [this database do-sync-fn]
"This function is basically around-advice for `sync-database!` and `sync-table!` operations.
Implementers can setup any context necessary for syncing, then need to call DO-SYNC-FN,
which takes no args.
* `:driver-name`
(sync-in-context [_ database do-sync-fn]
A human-readable string naming the DB this driver works with, e.g. `\"PostgreSQL\"`.
* `:details-fields`
A vector of maps that contain information about connection properties that should
be exposed to the user for databases that will use this driver. This information is used to build the UI for editing
a `Database` `details` map, and for validating it on the Backend. It should include things like `host`,
`port`, and other driver-specific parameters. Each field information map should have the following properties:
* `:name`
The key that should be used to store this property in the `details` map.
* `:display-name`
Human-readable name that should be displayed to the User in UI for editing this field.
* `:type` *(OPTIONAL)*
`:string`, `:integer`, `:boolean`, or `:password`. Defaults to `:string`.
* `:default` *(OPTIONAL)*
A default value for this field if the user hasn't set an explicit value. This is shown in the UI as a placeholder.
* `:placeholder` *(OPTIONAL)*
Placeholder value to show in the UI if user hasn't set an explicit value. Similar to `:default`, but this value is
*not* saved to `:details` if no explicit value is set. Since `:default` values are also shown as placeholders, you
cannot specify both `:default` and `:placeholder`.
* `:required` *(OPTIONAL)*
Is this property required? Defaults to `false`.
* `:features` *(OPTIONAL)*
A set of keyword names of optional features supported by this driver, such as `:foreign-keys`.
#### FUNCTIONS
* `(can-connect? [details-map])`
Check whether we can connect to a `Database` with DETAILS-MAP and perform a simple query. For example, a SQL database might
try running a query like `SELECT 1;`. This function should return `true` or `false`.
* `(active-table-names [database])`
Return a set of string names of tables, collections, or equivalent that currently exist in DATABASE.
* `(active-column-names->type [table])`
Return a map of string names of active columns (or equivalent) -> `Field` `base_type` for TABLE (or equivalent).
* `(table-pks [table])`
Return a set of string names of active Fields that are primary keys for TABLE (or equivalent).
* `(field-values-lazy-seq [field])`
Return a lazy sequence of all values of FIELD.
This is used to implement `mark-json-field!`, and fallback implentations of `mark-no-preview-display-field!` and `mark-url-field!`
if drivers *don't* implement `field-avg-length` and `field-percent-urls`, respectively.
* `(process-query [query])`
Process a native or structured QUERY. This function is called by `metabase.driver/process-query` after performing various driver-unspecific
steps like Query Expansion and other preprocessing.
* `(table-fks [table])` *(REQUIRED FOR DRIVERS THAT SUPPORT `:foreign-keys`)*
Return a set of maps containing info about FK columns for TABLE.
Each map should contain the following keys:
* `fk-column-name`
* `dest-table-name`
* `dest-column-name`
* `(active-nested-field-name->type [field])` *(REQUIRED FOR DRIVERS THAT SUPPORT `:nested-fields`)*
Return a map of string names of active child `Fields` of FIELD -> `Field.base_type`.
* `(humanize-connection-error-message [message])` *(OPTIONAL)*
Return a humanized (user-facing) version of an connection error message string.
Generic error messages are provided in the constant `connection-error-messages`; return one of these whenever possible.
* `(sync-in-context [database f])` *(OPTIONAL)*
Drivers may provide this function if they need to do special setup before a sync operation such as `sync-database!`. The sync
operation itself is encapsulated as the lambda F, which must be called with no arguments.
(defn sync-in-context [database f]
(with-jdbc-metadata [_ database]
(do-sync-fn)))")
(active-table-names [this database]
"Return a set of string names of tables, collections, or equivalent that currently exist in DATABASE.")
(active-column-names->type [this table]
"Return a map of string names of active columns (or equivalent) -> `Field` `base_type` for TABLE (or equivalent).")
(table-pks [this table]
"Return a set of string names of active Fields that are primary keys for TABLE (or equivalent).")
(field-values-lazy-seq [this field]
"Return a lazy sequence of all values of Field.
This is used to implement `mark-json-field!`, and fallback implentations of `mark-no-preview-display-field!` and `mark-url-field!`
if drivers *don't* implement `ISyncDriverFieldAvgLength` or `ISyncDriverFieldPercentUrls`, respectively.")
(table-rows-seq [this database table-name]
"Return a sequence of all the rows in a table with a given TABLE-NAME.
Currently, this is only used for iterating over the values in a `_metabase_metadata` table. As such, the results are not expected to be returned lazily.
(table-rows-seq driver (Database 2) \"_metabase_metadata\")
-> [{:keypath \"people.description\"
:value \"...\"}
...]")
;; Query Processing
(process-query [this query]
"Process a native or structured query.
(Don't use this directly; instead, use `metabase.driver/process-query`,
which does things like preprocessing before calling the appropriate implementation.)")
(wrap-process-query-middleware [this qp-fn]
"Custom QP middleware for this driver.
Like `sync-in-context`, but for running queries rather than syncing. This is basically around-advice for the QP pre and post-processing stages.
This should be used to do things like open DB connections that need to remain open for the duration of post-processing.
This middleware is injected into the QP middleware stack immediately after the Query Expander; in other words, it will receive the expanded query.
See the Mongo driver for and example of how this is intended to be used."))
;; ## ISyncDriverTableFKs Protocol (Optional)
(defprotocol ISyncDriverTableFKs
"Optional protocol to provide FK information for a TABLE.
If a sync driver implements it, Table FKs will be synced; otherwise, the step will be skipped."
(table-fks [this table]
"Return a set of maps containing info about FK columns for TABLE.
Each map should contain the following keys:
* fk-column-name
* dest-table-name
* dest-column-name"))
(defprotocol ISyncDriverFieldNestedFields
"Optional protocol that should provide information about the subfields of a FIELD when applicable.
Drivers that declare support for `:nested-fields` should implement this protocol."
(active-nested-field-name->type [this field]
"Return a map of string names of active child `Fields` of FIELD -> `Field.base_type`."))
;; ## ISyncDriverField Protocols (Optional)
;; These are optional protocol that drivers can implement to be used instead of falling back to field-values-lazy-seq for certain Field
;; syncing operations, which involves iterating over a few thousand values of the Field in Clojure-land. Since that's slower, it's
;; preferable to provide implementations of ISyncDriverFieldAvgLength/ISyncDriverFieldPercentUrls when possible.
(defprotocol ISyncDriverFieldAvgLength
"Optional. If this isn't provided, a fallback implementation that calculates average length in Clojure-land will be used instead."
(field-avg-length [this field]
"Return the average length of all non-nil values of textual FIELD."))
(defprotocol ISyncDriverFieldPercentUrls
"Optional. If this isn't provided, a fallback implementation that calculates URL percentage in Clojure-land will be used instead."
(field-percent-urls [this field]
"Return the percentage of non-nil values of textual FIELD that are valid URLs."))
;;; ## ISyncDriverSpecificSyncField (Optional)
(defprotocol ISyncDriverSpecificSyncField
"Optional. Do driver-specific syncing for a FIELD."
(driver-specific-sync-field! [this field]
"This is a chance for drivers to do custom Field syncing specific to their database.
For example, the Postgres driver can mark Postgres JSON fields as `special_type = json`.
As with the other Field syncing functions in `metabase.driver.sync`, this method should return the modified
FIELD, if any, or `nil`."))
;; ## Helper Functions
(def ^:private valid-feature? (partial contains? driver-optional-features))
(defn supports?
"Does DRIVER support FEATURE?"
[{:keys [features]} ^Keyword feature]
{:pre [(set? features)
(every? valid-feature? features)
(valid-feature? feature)]}
(contains? features feature))
(defn assert-driver-supports
"Helper fn. Assert that DRIVER supports FEATURE."
[driver ^Keyword feature]
(when-not (supports? driver feature)
(throw (Exception. (format "%s is not supported by this driver." (name feature))))))
(f)))
* `(process-query-in-context [f])` *(OPTIONAL)*
Similar to `sync-in-context`, but for running queries rather than syncing. This should be used to do things like open DB connections
that need to remain open for the duration of post-processing. This function follows a middleware pattern and is injected into the QP
middleware stack immediately after the Query Expander; in other words, it will receive the expanded query.
See the Mongo and H2 drivers for examples of how this is intended to be used.
* `(table-rows-seq [database table-name])` *(OPTIONAL)*
Return a sequence of all the rows in a table with a given TABLE-NAME.
Currently, this is only used for iterating over the values in a `_metabase_metadata` table. As such, the results are not expected to be returned lazily.
* `(field-avg-length [field])` *(OPTIONAL)*
If possible, provide an efficent DB-level function to calculate the average length of non-nil values of textual FIELD, which is used to determine whether a `Field`
should be marked as a `:category`. If this function is not provided, a fallback implementation that iterates over results in Clojure-land is used instead.
* `(field-percent-urls [field])` *(OPTIONAL)*
If possible, provide an efficent DB-level function to calculate what percentage of non-nil values of textual FIELD are valid URLs, which is used to determine
whether a `Field` should be marked as a `:url`. If this function is not provided, a fallback implementation that iterates over results in Clojure-land is used instead.
* `(driver-specific-sync-field! [field])` *(OPTIONAL)*
This is a chance for drivers to do custom `Field` syncing specific to their database.
For example, the Postgres driver can mark Postgres JSON fields as `special_type = json`.
As with the other Field syncing functions in `metabase.driver.sync`, this method should return the modified FIELD, if any, or `nil`."
[driver-name driver-map]
`(def ~(vary-meta driver-name assoc :metabase.driver/driver (keyword driver-name))
(let [m# ~driver-map]
(verify-driver m#)
m#)))
src/metabase/driver/mongo.clj
+ 44
− 36
View file @ 3d9b139b
......@@ -12,7 +12,7 @@
[db :as mdb]
[query :as mq])
[metabase.driver :as driver]
[metabase.driver.interface :as i, :refer [IDriver ISyncDriverFieldNestedFields]]
[metabase.driver.interface :as i, :refer [defdriver]]
(metabase.driver.mongo [query-processor :as qp]
[util :refer [*mongo-connection* with-mongo-connection values->base-type]])
[metabase.util :as u]))
......@@ -38,22 +38,19 @@
{:pre [(map? field)]
:post [(keyword? %)]}
(with-mongo-connection [_ @(:db @(:table field))]
(values->base-type (field-values-lazy-seq driver field))))
(values->base-type (field-values-lazy-seq field))))
;;; ## MongoDriver
(defn- can-connect? [_ database]
(with-mongo-connection [^com.mongodb.DB conn database]
(defn- can-connect? [details]
(with-mongo-connection [^com.mongodb.DB conn details]
(= (-> (cmd/db-stats conn)
(conv/from-db-object :keywordize)
:ok)
1.0)))
(defn- can-connect-with-details? [this details]
(can-connect? this {:details details}))
(defn- humanize-connection-error-message [_ message]
(defn- humanize-connection-error-message [message]
(condp re-matches message
#"^Timed out after \d+ ms while waiting for a server .*$"
(i/connection-error-messages :cannot-connect-check-host-and-port)
......@@ -64,28 +61,28 @@
#"^Password can not be null when the authentication mechanism is unspecified$"
(i/connection-error-messages :password-required)
#".*" ; default
#".*" ; default
message))
(defn- wrap-process-query-middleware [_ qp]
(defn- process-query-in-context [qp]
(fn [query]
(with-mongo-connection [^com.mongodb.DB conn (:database query)]
(qp query))))
(defn- process-query [_ query]
(defn- process-query [query]
(qp/process-and-run query))
;;; ### Syncing
(defn- sync-in-context [_ database do-sync-fn]
(defn- sync-in-context [database do-sync-fn]
(with-mongo-connection [_ database]
(do-sync-fn)))
(defn- active-table-names [_ database]
(defn- active-table-names [database]
(with-mongo-connection [^com.mongodb.DB conn database]
(-> (mdb/get-collection-names conn)
(set/difference #{"system.indexes"}))))
(defn- active-column-names->type [_ table]
(defn- active-column-names->type [table]
(with-mongo-connection [_ @(:db table)]
(into {} (for [column-name (table->column-names table)]
{(name column-name)
......@@ -93,7 +90,7 @@
:table (delay table)
:qualified-name-components (delay [(:name table) (name column-name)])})}))))
(defn- field-values-lazy-seq [_ {:keys [qualified-name-components table], :as field}]
(defn- field-values-lazy-seq [{:keys [qualified-name-components table], :as field}]
(assert (and (map? field)
(delay? qualified-name-components)
(delay? table))
......@@ -108,11 +105,11 @@
(mq/with-collection *mongo-connection* (:name table)
(mq/fields [(apply str (interpose "." name-components))]))))))
(defn- active-nested-field-name->type [this field]
(defn- active-nested-field-name->type [field]
;; Build a map of nested-field-key -> type -> count
;; TODO - using an atom isn't the *fastest* thing in the world (but is the easiest); consider alternate implementation
(let [field->type->count (atom {})]
(doseq [val (take i/max-sync-lazy-seq-results (field-values-lazy-seq this field))]
(doseq [val (take i/max-sync-lazy-seq-results (field-values-lazy-seq field))]
(when (map? val)
(doseq [[k v] val]
(swap! field->type->count update-in [k (type v)] #(if % (inc %) 1)))))
......@@ -125,22 +122,33 @@
first ; keep just the type
driver/class->base-type))))))
(defrecord MongoDriver [])
(extend MongoDriver
IDriver {:can-connect? can-connect?
:can-connect-with-details? can-connect-with-details?
:humanize-connection-error-message humanize-connection-error-message
:wrap-process-query-middleware wrap-process-query-middleware
:process-query process-query
:sync-in-context sync-in-context
:active-table-names active-table-names
:active-column-names->type active-column-names->type
:table-pks (constantly #{"_id"})
:field-values-lazy-seq field-values-lazy-seq}
ISyncDriverFieldNestedFields {:active-nested-field-name->type active-nested-field-name->type})
(def driver
"Concrete instance of the MongoDB driver."
(map->MongoDriver {:features #{:nested-fields}}))
(defdriver mongo
{:driver-name "MongoDB"
:details-fields [{:name "host"
:display-name "Host"
:default "localhost"}
{:name "port"
:display-name "Port"
:type :integer
:default 27017}
{:name "dbname"
:display-name "Database name"
:placeholder "carrierPigeonDeliveries"
:required true}
{:name "user"
:display-name "Database username"
:placeholder "What username do you use to login to the database?"}
{:name "pass"
:display-name "Database password"
:type :password
:placeholder "******"}]
:features #{:nested-fields}
:can-connect? can-connect?
:active-table-names active-table-names
:field-values-lazy-seq field-values-lazy-seq
:active-column-names->type active-column-names->type
:table-pks (constantly #{"_id"})
:process-query process-query
:process-query-in-context process-query-in-context
:sync-in-context sync-in-context
:active-nested-field-name->type active-nested-field-name->type})
src/metabase/driver/mongo/query_processor.clj
+ 2
− 4
View file @ 3d9b139b
......@@ -12,9 +12,7 @@
[operators :refer :all]
[query :refer :all])
[metabase.db :refer :all]
[metabase.driver :as driver]
(metabase.driver [interface :as i]
[query-processor :as qp])
[metabase.driver.query-processor :as qp]
[metabase.driver.query-processor.interface :refer [qualified-name-components]]
[metabase.driver.mongo.util :refer [with-mongo-connection *mongo-connection* values->base-type]]
[metabase.models.field :as field]
......@@ -162,7 +160,7 @@
(constantly true))
field-id (or (:field-id field) ; Field
(:field-id (:field field)))] ; DateTimeField
(->> (i/field-values-lazy-seq @(ns-resolve 'metabase.driver.mongo 'driver) (sel :one field/Field :id field-id)) ; resolve driver at runtime to avoid circular deps
(->> (@(resolve 'metabase.driver.mongo/field-values-lazy-seq) (sel :one field/Field :id field-id)) ; resolve driver at runtime to avoid circular deps
(filter identity)
(map hash)
(map #(conj! values %))
......
src/metabase/driver/mysql.clj
+ 40
− 34
View file @ 3d9b139b
......@@ -6,12 +6,9 @@
mysql)
(korma.sql [engine :refer [sql-func]]
[utils :as utils])
(metabase.driver [generic-sql :as generic-sql, :refer [GenericSQLIDriverMixin GenericSQLISyncDriverTableFKsMixin
GenericSQLISyncDriverFieldAvgLengthMixin GenericSQLISyncDriverFieldPercentUrlsMixin]]
[interface :as i, :refer [IDriver ISyncDriverTableFKs ISyncDriverFieldAvgLength ISyncDriverFieldPercentUrls
ISyncDriverSpecificSyncField driver-specific-sync-field!]])
(metabase.driver.generic-sql [interface :refer [ISqlDriverDatabaseSpecific]]
[util :refer [funcs]])))
(metabase.driver [generic-sql :refer [sql-driver]]
[interface :as i, :refer [defdriver]])
[metabase.driver.generic-sql.util :refer [funcs]]))
;;; # Korma 0.4.2 Bug Workaround
;; (Buggy code @ https://github.com/korma/Korma/blob/684178c386df529558bbf82097635df6e75fb339/src/korma/mysql.clj)
......@@ -61,21 +58,18 @@
:VARCHAR :TextField
:YEAR :IntegerField})
(defn- connection-details->connection-spec [_ details]
(defn- connection-details->spec [details]
(-> details
(set/rename-keys {:dbname :db})
kdb/mysql))
(defn- database->connection-details [_ {:keys [details]}]
details)
(defn- unix-timestamp->timestamp [_ field-or-value seconds-or-milliseconds]
(defn- unix-timestamp->timestamp [field-or-value seconds-or-milliseconds]
(utils/func (case seconds-or-milliseconds
:seconds "FROM_UNIXTIME(%s)"
:milliseconds "FROM_UNIXTIME(%s / 1000)")
[field-or-value]))
(defn- timezone->set-timezone-sql [_ timezone]
(defn- timezone->set-timezone-sql [timezone]
;; If this fails you need to load the timezone definitions from your system into MySQL;
;; run the command `mysql_tzinfo_to_sql /usr/share/zoneinfo | mysql -u root mysql`
;; See https://dev.mysql.com/doc/refman/5.7/en/time-zone-support.html for details
......@@ -97,7 +91,7 @@
["((QUARTER(%s) * 3) - 2)" field-or-value]
(k/raw "'-01'")]))
(defn- date [_ unit field-or-value]
(defn- date [unit field-or-value]
(if (= unit :quarter)
(trunc-to-quarter field-or-value)
(utils/func (case unit
......@@ -121,7 +115,7 @@
:year "YEAR(%s)")
[field-or-value])))
(defn- date-interval [_ unit amount]
(defn- date-interval [unit amount]
(utils/generated (format (case unit
:minute "DATE_ADD(NOW(), INTERVAL %d MINUTE)"
:hour "DATE_ADD(NOW(), INTERVAL %d HOUR)"
......@@ -132,7 +126,7 @@
:year "DATE_ADD(NOW(), INTERVAL %d YEAR)")
amount)))
(defn- humanize-connection-error-message [_ message]
(defn- humanize-connection-error-message [message]
(condp re-matches message
#"^Communications link failure\s+The last packet sent successfully to the server was 0 milliseconds ago. The driver has not received any packets from the server.$"
(i/connection-error-messages :cannot-connect-check-host-and-port)
......@@ -149,22 +143,34 @@
#".*" ; default
message))
(defrecord MySQLDriver [])
(extend MySQLDriver
ISqlDriverDatabaseSpecific {:connection-details->connection-spec connection-details->connection-spec
:database->connection-details database->connection-details
:unix-timestamp->timestamp unix-timestamp->timestamp
:date date
:date-interval date-interval
:timezone->set-timezone-sql timezone->set-timezone-sql}
IDriver (assoc GenericSQLIDriverMixin
:humanize-connection-error-message humanize-connection-error-message)
ISyncDriverTableFKs GenericSQLISyncDriverTableFKsMixin
ISyncDriverFieldAvgLength GenericSQLISyncDriverFieldAvgLengthMixin
ISyncDriverFieldPercentUrls GenericSQLISyncDriverFieldPercentUrlsMixin)
(def ^:const driver
(map->MySQLDriver {:column->base-type column->base-type
:features (conj generic-sql/features :set-timezone)
:sql-string-length-fn :CHAR_LENGTH}))
(defdriver mysql
(-> {:driver-name "MySQL"
:details-fields [{:name "host"
:display-name "Host"
:default "localhost"}
{:name "port"
:display-name "Port"
:type :integer
:default 3306}
{:name "dbname"
:display-name "Database name"
:placeholder "birds_of_the_word"
:required true}
{:name "user"
:display-name "Database username"
:placeholder "What username do you use to login to the database?"
:required true}
{:name "password"
:display-name "Database password"
:type :password
:placeholder "*******"}]
:column->base-type column->base-type
:sql-string-length-fn :CHAR_LENGTH
:connection-details->spec connection-details->spec
:unix-timestamp->timestamp unix-timestamp->timestamp
:date date
:date-interval date-interval
:timezone->set-timezone-sql timezone->set-timezone-sql
:humanize-connection-error-message humanize-connection-error-message}
sql-driver
(update :features conj :set-timezone)))
src/metabase/driver/postgres.clj
+ 49
− 44
View file @ 3d9b139b
......@@ -9,14 +9,9 @@
[swiss.arrows :refer :all]
[metabase.db :refer [upd]]
[metabase.models.field :refer [Field]]
[metabase.driver :as driver]
(metabase.driver [generic-sql :as generic-sql, :refer [GenericSQLIDriverMixin GenericSQLISyncDriverTableFKsMixin
GenericSQLISyncDriverFieldAvgLengthMixin GenericSQLISyncDriverFieldPercentUrlsMixin]]
[interface :as i, :refer [IDriver ISyncDriverTableFKs ISyncDriverFieldAvgLength ISyncDriverFieldPercentUrls
ISyncDriverSpecificSyncField]])
[metabase.driver.generic-sql :as generic-sql]
(metabase.driver.generic-sql [interface :refer [ISqlDriverDatabaseSpecific]]
[util :refer [with-jdbc-metadata]]))
(metabase.driver [generic-sql :refer [sql-driver]]
[interface :as i, :refer [defdriver]])
[metabase.driver.generic-sql.util :refer [with-jdbc-metadata]])
;; This is necessary for when NonValidatingFactory is passed in the sslfactory connection string argument,
;; e.x. when connecting to a Heroku Postgres database from outside of Heroku.
(:import org.postgresql.ssl.NonValidatingFactory))
......@@ -88,34 +83,28 @@
:sslmode "require"
:sslfactory "org.postgresql.ssl.NonValidatingFactory"}) ; HACK Why enable SSL if we disable certificate validation?
(defn- connection-details->connection-spec [_ {:keys [ssl] :as details-map}]
(defn- connection-details->spec [{:keys [ssl] :as details-map}]
(-> details-map
(update :port (fn [port]
(if (string? port) (Integer/parseInt port)
port)))
(dissoc :ssl) ; remove :ssl in case it's false; DB will still try (& fail) to connect if the key is there
(merge (when ssl ; merging ssl-params will add :ssl back in if desirable
ssl-params))
(rename-keys {:dbname :db})
kdb/postgres))
(defn- database->connection-details [_ {:keys [details]}]
(let [{:keys [host port]} details]
(-> details
(assoc :host host
:ssl (:ssl details)
:port (if (string? port) (Integer/parseInt port)
port))
(rename-keys {:dbname :db}))))
(defn- unix-timestamp->timestamp [_ field-or-value seconds-or-milliseconds]
(defn- unix-timestamp->timestamp [field-or-value seconds-or-milliseconds]
(utils/func (case seconds-or-milliseconds
:seconds "TO_TIMESTAMP(%s)"
:milliseconds "TO_TIMESTAMP(%s / 1000)")
[field-or-value]))
(defn- timezone->set-timezone-sql [_ timezone]
(defn- timezone->set-timezone-sql [timezone]
(format "SET LOCAL timezone TO '%s';" timezone))
(defn- driver-specific-sync-field! [_ {:keys [table], :as field}]
(defn- driver-specific-sync-field! [{:keys [table], :as field}]
(with-jdbc-metadata [^java.sql.DatabaseMetaData md @(:db @table)]
(let [[{:keys [type_name]}] (->> (.getColumns md nil nil (:name @table) (:name field))
jdbc/result-set-seq)]
......@@ -123,7 +112,7 @@
(upd Field (:id field) :special_type :json)
(assoc field :special_type :json)))))
(defn- date [_ unit field-or-value]
(defn- date [unit field-or-value]
(utils/func (case unit
:default "CAST(%s AS TIMESTAMP)"
:minute "DATE_TRUNC('minute', %s)"
......@@ -145,7 +134,7 @@
:year "CAST(EXTRACT(YEAR FROM %s) AS INTEGER)")
[field-or-value]))
(defn- date-interval [_ unit amount]
(defn- date-interval [unit amount]
(utils/generated (format (case unit
:minute "(NOW() + INTERVAL '%d minute')"
:hour "(NOW() + INTERVAL '%d hour')"
......@@ -156,7 +145,7 @@
:year "(NOW() + INTERVAL '%d year')")
amount)))
(defn- humanize-connection-error-message [_ message]
(defn- humanize-connection-error-message [message]
(condp re-matches message
#"^FATAL: database \".*\" does not exist$"
(i/connection-error-messages :database-name-incorrect)
......@@ -180,23 +169,39 @@
#".*" ; default
message))
(defrecord PostgresDriver [])
(extend PostgresDriver
ISqlDriverDatabaseSpecific {:connection-details->connection-spec connection-details->connection-spec
:database->connection-details database->connection-details
:unix-timestamp->timestamp unix-timestamp->timestamp
:date date
:date-interval date-interval
:timezone->set-timezone-sql timezone->set-timezone-sql}
ISyncDriverSpecificSyncField {:driver-specific-sync-field! driver-specific-sync-field!}
IDriver (assoc GenericSQLIDriverMixin
:humanize-connection-error-message humanize-connection-error-message)
ISyncDriverTableFKs GenericSQLISyncDriverTableFKsMixin
ISyncDriverFieldAvgLength GenericSQLISyncDriverFieldAvgLengthMixin
ISyncDriverFieldPercentUrls GenericSQLISyncDriverFieldPercentUrlsMixin)
(def ^:const driver
(map->PostgresDriver {:column->base-type column->base-type
:features (conj generic-sql/features :set-timezone)
:sql-string-length-fn :CHAR_LENGTH}))
(defdriver postgres
(-> {:driver-name "PostgreSQL"
:details-fields [{:name "host"
:display-name "Host"
:default "localhost"}
{:name "port"
:display-name "Port"
:type :integer
:default 5432}
{:name "dbname"
:display-name "Database name"
:placeholder "birds_of_the_word"
:required true}
{:name "user"
:display-name "Database username"
:placeholder "What username do you use to login to the database?"
:required true}
{:name "password"
:display-name "Database password"
:type :password
:placeholder "*******"}
{:name "ssl"
:display-name "Use a secure connection (SSL)?"
:type :boolean
:default false}]
:sql-string-length-fn :CHAR_LENGTH
:column->base-type column->base-type
:connection-details->spec connection-details->spec
:unix-timestamp->timestamp unix-timestamp->timestamp
:date date
:date-interval date-interval
:timezone->set-timezone-sql timezone->set-timezone-sql
:driver-specific-sync-field! driver-specific-sync-field!
:humanize-connection-error-message humanize-connection-error-message}
sql-driver
(update :features conj :set-timezone)))
src/metabase/driver/query_processor.clj
+ 8
− 7
View file @ 3d9b139b
......@@ -8,7 +8,6 @@
[medley.core :as m]
[swiss.arrows :refer [<<-]]
[metabase.db :refer :all]
[metabase.driver.interface :as i]
(metabase.driver.query-processor [annotate :as annotate]
[expand :as expand]
[interface :refer :all]
......@@ -193,10 +192,10 @@
(fn [f]
(if-not (map? f) f
(m/filter-vals identity (into {} f))))
;; obscure DB details when logging. Just log the class of driver because we don't care about its properties
;; obscure DB details when logging. Just log the name of driver because we don't care about its properties
(-> query
(assoc-in [:database :details] "😋 ") ; :yum:
(update :driver class)))))))
(update :driver :driver-name)))))))
(qp query)))
......@@ -241,8 +240,9 @@
(qp query))))
(defn- process-structured [{:keys [driver], :as query}]
(let [driver-process-query (partial i/process-query driver)
driver-wrap-process-query (partial i/wrap-process-query-middleware driver)]
(let [driver-process-query (:process-query driver)
driver-wrap-process-query (or (:process-query-in-context driver)
(fn [qp] qp))]
((<<- wrap-catch-exceptions
pre-expand
driver-wrap-process-query
......@@ -257,8 +257,9 @@
driver-process-query) query)))
(defn- process-native [{:keys [driver], :as query}]
(let [driver-process-query (partial i/process-query driver)
driver-wrap-process-query (partial i/wrap-process-query-middleware driver)]
(let [driver-process-query (:process-query driver)
driver-wrap-process-query (or (:process-query-in-context driver)
(fn [qp] qp))]
((<<- wrap-catch-exceptions
driver-wrap-process-query
post-add-row-count-and-status
......
src/metabase/driver/query_processor/expand.clj
+ 2
− 3
View file @ 3d9b139b
......@@ -24,15 +24,14 @@
;; ## -------------------- Expansion - Impl --------------------
(def ^:private ^:dynamic *original-query-dict*
"The entire original Query dict being expanded."
nil)
(defn- assert-driver-supports [^Keyword feature]
{:pre [(:driver *original-query-dict*)]}
(driver/assert-driver-supports (:driver *original-query-dict*) feature))
(when-not (contains? (:features (:driver *original-query-dict*)) feature)
(throw (Exception. (format "%s is not supported by this driver." (name feature))))))
(defn- non-empty-clause? [clause]
(and clause
......
src/metabase/driver/sync.clj
+ 125
− 117
View file @ 3d9b139b
......@@ -8,7 +8,7 @@
[korma.core :as k]
[medley.core :as m]
[metabase.db :refer :all]
(metabase.driver [interface :refer :all]
(metabase.driver [interface :refer [max-sync-lazy-seq-results]]
[query-processor :as qp])
[metabase.driver.sync.queries :as queries]
[metabase.events :as events]
......@@ -38,56 +38,60 @@
;; ## sync-database! and sync-table!
(defn -sync-database! [{:keys [active-table-names], :as driver} database]
(let [start-time (System/currentTimeMillis)
tracking-hash (str (java.util.UUID/randomUUID))]
(log/info (u/format-color 'magenta "Syncing %s database '%s'..." (name (:engine database)) (:name database)))
(events/publish-event :database-sync-begin {:database_id (:id database) :custom_id tracking-hash})
(let [active-table-names (active-table-names database)
table-name->id (sel :many :field->id [Table :name] :db_id (:id database) :active true)]
(assert (set? active-table-names) "active-table-names should return a set.")
(assert (every? string? active-table-names) "active-table-names should return the names of Tables as *strings*.")
;; First, let's mark any Tables that are no longer active as such.
;; These are ones that exist in table-name->id but not in active-table-names.
(doseq [[table-name table-id] table-name->id]
(when-not (contains? active-table-names table-name)
(upd Table table-id :active false)
(log/info (u/format-color 'cyan "Marked table %s.%s as inactive." (:name database) table-name))
;; We need to mark driver Table's Fields as inactive so we don't expose them in UI such as FK selector (etc.)
(k/update Field
(k/where {:table_id table-id})
(k/set-fields {:active false}))))
;; Next, we'll create new Tables (ones that came back in active-table-names but *not* in table-name->id)
(let [existing-table-names (set (keys table-name->id))
new-table-names (set/difference active-table-names existing-table-names)]
(when (seq new-table-names)
(log/debug (u/format-color 'blue "Found new tables: %s" new-table-names))
(doseq [new-table-name new-table-names]
;; If it's a _metabase_metadata table then we'll handle later once everything else has been synced
(when-not (= (s/lower-case new-table-name) "_metabase_metadata")
(ins Table :db_id (:id database), :active true, :name new-table-name))))))
;; Now sync the active tables
(->> (sel :many Table :db_id (:id database) :active true)
(map #(assoc % :db (delay database))) ; replace default delays with ones that reuse database (and don't require a DB call)
(sync-database-active-tables! driver))
;; Ok, now if we had a _metabase_metadata table from earlier we can go ahead and sync from it
(sync-metabase-metadata-table! driver database)
(events/publish-event :database-sync-end {:database_id (:id database) :custom_id tracking-hash :running_time (- (System/currentTimeMillis) start-time)})
(log/info (u/format-color 'magenta "Finished syncing %s database %s. (%d ms)" (name (:engine database)) (:name database)
(- (System/currentTimeMillis) start-time)))))
(defn sync-database!
"Sync DATABASE and all its Tables and Fields."
[driver database]
[{:keys [sync-in-context], :as driver} database]
(binding [qp/*disable-qp-logging* true
*sel-disable-logging* true]
(sync-in-context driver database
(fn []
(let [start-time (System/currentTimeMillis)
tracking-hash (str (java.util.UUID/randomUUID))]
(log/info (u/format-color 'magenta "Syncing %s database '%s'..." (name (:engine database)) (:name database)))
(events/publish-event :database-sync-begin {:database_id (:id database) :custom_id tracking-hash})
(let [active-table-names (active-table-names driver database)
table-name->id (sel :many :field->id [Table :name] :db_id (:id database) :active true)]
(assert (set? active-table-names) "active-table-names should return a set.")
(assert (every? string? active-table-names) "active-table-names should return the names of Tables as *strings*.")
;; First, let's mark any Tables that are no longer active as such.
;; These are ones that exist in table-name->id but not in active-table-names.
(doseq [[table-name table-id] table-name->id]
(when-not (contains? active-table-names table-name)
(upd Table table-id :active false)
(log/info (u/format-color 'cyan "Marked table %s.%s as inactive." (:name database) table-name))
;; We need to mark driver Table's Fields as inactive so we don't expose them in UI such as FK selector (etc.)
(k/update Field
(k/where {:table_id table-id})
(k/set-fields {:active false}))))
;; Next, we'll create new Tables (ones that came back in active-table-names but *not* in table-name->id)
(let [existing-table-names (set (keys table-name->id))
new-table-names (set/difference active-table-names existing-table-names)]
(when (seq new-table-names)
(log/debug (u/format-color 'blue "Found new tables: %s" new-table-names))
(doseq [new-table-name new-table-names]
;; If it's a _metabase_metadata table then we'll handle later once everything else has been synced
(when-not (= (s/lower-case new-table-name) "_metabase_metadata")
(ins Table :db_id (:id database), :active true, :name new-table-name))))))
;; Now sync the active tables
(->> (sel :many Table :db_id (:id database) :active true)
(map #(assoc % :db (delay database))) ; replace default delays with ones that reuse database (and don't require a DB call)
(sync-database-active-tables! driver))
;; Ok, now if we had a _metabase_metadata table from earlier we can go ahead and sync from it
(sync-metabase-metadata-table! driver database)
(events/publish-event :database-sync-end {:database_id (:id database) :custom_id tracking-hash :running_time (- (System/currentTimeMillis) start-time)})
(log/info (u/format-color 'magenta "Finished syncing %s database %s. (%d ms)" (name (:engine database)) (:name database)
(- (System/currentTimeMillis) start-time))))))))
(let [f (partial -sync-database! driver database)]
(if sync-in-context
(sync-in-context database f)
(f)))))
(defn- sync-metabase-metadata-table!
"Databases may include a table named `_metabase_metadata` (case-insentive) which includes descriptions or other metadata about the `Tables` and `Fields`
......@@ -103,35 +107,38 @@
`keypath` is of the form `table-name.key` or `table-name.field-name.key`, where `key` is the name of some property of `Table` or `Field`.
This functionality is currently only used by the Sample Dataset."
[driver database]
(doseq [table-name (active-table-names driver database)]
(when (= (s/lower-case table-name) "_metabase_metadata")
(doseq [{:keys [keypath value]} (table-rows-seq driver database table-name)]
(let [[_ table-name field-name k] (re-matches #"^([^.]+)\.(?:([^.]+)\.)?([^.]+)$" keypath)]
(try (when (not= 1 (if field-name
(k/update Field
(k/where {:name field-name, :table_id (k/subselect Table
(k/fields :id)
(k/where {:db_id (:id database), :name table-name}))})
(k/set-fields {(keyword k) value}))
(k/update Table
(k/where {:name table-name, :db_id (:id database)})
(k/set-fields {(keyword k) value}))))
(log/error (u/format-color "Error syncing _metabase_metadata: no matching keypath: %s" keypath)))
(catch Throwable e
(log/error (u/format-color 'red "Error in _metabase_metadata: %s" (.getMessage e))))))))))
This functionality is currently only used by the Sample Dataset. In order to use this functionality, drivers must implement optional fn `:table-rows-seq`."
[{:keys [table-rows-seq active-table-names]} database]
(when table-rows-seq
(doseq [table-name (active-table-names database)]
(when (= (s/lower-case table-name) "_metabase_metadata")
(doseq [{:keys [keypath value]} (table-rows-seq database table-name)]
(let [[_ table-name field-name k] (re-matches #"^([^.]+)\.(?:([^.]+)\.)?([^.]+)$" keypath)]
(try (when (not= 1 (if field-name
(k/update Field
(k/where {:name field-name, :table_id (k/subselect Table
(k/fields :id)
(k/where {:db_id (:id database), :name table-name}))})
(k/set-fields {(keyword k) value}))
(k/update Table
(k/where {:name table-name, :db_id (:id database)})
(k/set-fields {(keyword k) value}))))
(log/error (u/format-color "Error syncing _metabase_metadata: no matching keypath: %s" keypath)))
(catch Throwable e
(log/error (u/format-color 'red "Error in _metabase_metadata: %s" (.getMessage e)))))))))))
(defn sync-table!
"Sync a *single* TABLE by running all the sync steps for it.
This is used *instead* of `sync-database!` when syncing just one Table is desirable."
[driver table]
(let [database @(:db table)]
[{:keys [sync-in-context], :as driver} table]
(let [database @(:db table)
f (fn []
(sync-database-active-tables! driver [table])
(events/publish-event :table-sync {:table_id (:id table)}))]
(binding [qp/*disable-qp-logging* true]
(sync-in-context driver database
(fn []
(sync-database-active-tables! driver [table])
(events/publish-event :table-sync {:table_id (:id table)}))))))
(if sync-in-context
(sync-in-context database f)
(f)))))
;; ### sync-database-active-tables! -- runs the sync-table steps over sequence of Tables
......@@ -237,10 +244,10 @@
(defn- sync-table-active-fields-and-pks!
"Create new Fields (and mark old ones as inactive) for TABLE, and update PK fields."
[driver table]
[{:keys [active-column-names->type table-pks], :as driver} table]
(let [database @(:db table)]
;; Now do the syncing for Table's Fields
(let [active-column-names->type (active-column-names->type driver table)
(let [active-column-names->type (active-column-names->type table)
existing-field-name->field (sel :many :field->fields [Field :name :base_type :id], :table_id (:id table), :active true, :parent_id nil)]
(assert (map? active-column-names->type) "active-column-names->type should return a map.")
......@@ -274,7 +281,7 @@
;; TODO - we need to add functionality to update nested Field base types as well!
;; Now mark PK fields as such if needed
(let [pk-fields (table-pks driver table)]
(let [pk-fields (table-pks table)]
(u/try-apply update-table-pks! table pk-fields)))))
......@@ -292,9 +299,9 @@
(if (= field-count field-distinct-count) :1t1
:Mt1)))
(defn- sync-table-fks! [driver table]
(when (extends? ISyncDriverTableFKs (type driver))
(let [fks (table-fks driver table)]
(defn- sync-table-fks! [{:keys [features table-fks]} table]
(when (contains? features :foreign-keys)
(let [fks (table-fks table)]
(assert (and (set? fks)
(every? map? fks)
(every? :fk-column-name fks)
......@@ -364,9 +371,9 @@
(defn- maybe-driver-specific-sync-field!
"If driver implements `ISyncDriverSpecificSyncField`, call `driver-specific-sync-field!`."
[driver field]
(when (satisfies? ISyncDriverSpecificSyncField driver)
(driver-specific-sync-field! driver field)))
[{:keys [driver-specific-sync-field!]} field]
(when driver-specific-sync-field!
(driver-specific-sync-field! field)))
;; ### set-field-display-name-if-needed!
......@@ -399,27 +406,29 @@
(inc non-nil-count)
more)))))
(extend-protocol ISyncDriverFieldPercentUrls ; Default implementation
Object
(field-percent-urls [this field]
(let [field-values (->> (field-values-lazy-seq this field)
(filter identity)
(take max-sync-lazy-seq-results))]
(percent-valid-urls field-values))))
(defn- default-field-percent-urls
"Default implementation for optional driver fn `:field-percent-urls` that calculates percentage in Clojure-land."
[{:keys [field-values-lazy-seq]} field]
(->> (field-values-lazy-seq field)
(filter identity)
(take max-sync-lazy-seq-results)
percent-valid-urls))
(defn- mark-url-field!
"If FIELD is texual, doesn't have a `special_type`, and its non-nil values are primarily URLs, mark it as `special_type` `url`."
[driver field]
[{:keys [field-percent-urls], :as driver} field]
(when (and (not (:special_type field))
(contains? #{:CharField :TextField} (:base_type field)))
(when-let [percent-urls (field-percent-urls driver field)]
(assert (float? percent-urls))
(assert (>= percent-urls 0.0))
(assert (<= percent-urls 100.0))
(when (> percent-urls percent-valid-url-threshold)
(log/debug (u/format-color 'green "Field '%s' is %d%% URLs. Marking it as a URL." @(:qualified-name field) (int (math/round (* 100 percent-urls)))))
(upd Field (:id field) :special_type :url)
(assoc field :special_type :url)))))
(let [field-percent-urls (or field-percent-urls
(partial default-field-percent-urls driver))]
(when-let [percent-urls (field-percent-urls field)]
(assert (float? percent-urls))
(assert (>= percent-urls 0.0))
(assert (<= percent-urls 100.0))
(when (> percent-urls percent-valid-url-threshold)
(log/debug (u/format-color 'green "Field '%s' is %d%% URLs. Marking it as a URL." @(:qualified-name field) (int (math/round (* 100 percent-urls)))))
(upd Field (:id field) :special_type :url)
(assoc field :special_type :url))))))
;; ### mark-category-field-or-update-field-values!
......@@ -455,26 +464,26 @@
"Fields whose values' average length is greater than this amount should be marked as `preview_display = false`."
50)
(extend-protocol ISyncDriverFieldAvgLength ; Default implementation
Object
(field-avg-length [this field]
(let [field-values (->> (field-values-lazy-seq this field)
(filter identity)
(take max-sync-lazy-seq-results)) ; as with field-percent-urls it's probably fine to consider the first 10,000 values rather than potentially millions
field-values-count (count field-values)]
(if (= field-values-count 0) 0
(int (math/round (/ (->> field-values
(map str)
(map count)
(reduce +))
field-values-count)))))))
(defn- default-field-avg-length [{:keys [field-values-lazy-seq]} field]
(let [field-values (->> (field-values-lazy-seq field)
(filter identity)
(take max-sync-lazy-seq-results))
field-values-count (count field-values)]
(if (= field-values-count 0) 0
(int (math/round (/ (->> field-values
(map str)
(map count)
(reduce +))
field-values-count))))))
(defn- mark-no-preview-display-field!
"If FIELD's is textual and its average length is too great, mark it so it isn't displayed in the UI."
[driver field]
[{:keys [field-avg-length], :as driver} field]
(when (and (:preview_display field)
(contains? #{:CharField :TextField} (:base_type field)))
(let [avg-len (field-avg-length driver field)]
(let [field-avg-length (or field-avg-length
(partial default-field-avg-length driver))
avg-len (field-avg-length field)]
(assert (integer? avg-len) "field-avg-length should return an integer.")
(when (> avg-len average-length-no-preview-threshold)
(log/debug (u/format-color 'green "Field '%s' has an average length of %d. Not displaying it in previews." @(:qualified-name field) avg-len))
......@@ -506,10 +515,10 @@
(defn- mark-json-field!
"Mark FIELD as `:json` if it's textual, doesn't already have a special type, the majority of it's values are non-nil, and all of its non-nil values
are valid serialized JSON dictionaries or arrays."
[driver field]
[{:keys [field-values-lazy-seq]} field]
(when (and (not (:special_type field))
(contains? #{:CharField :TextField} (:base_type field))
(values-are-valid-json? (->> (field-values-lazy-seq driver field)
(values-are-valid-json? (->> (field-values-lazy-seq field)
(take max-sync-lazy-seq-results))))
(log/debug (u/format-color 'green "Field '%s' looks like it contains valid JSON objects. Setting special_type to :json." @(:qualified-name field)))
(upd Field (:id field) :special_type :json, :preview_display false)
......@@ -593,11 +602,10 @@
(assoc field :special_type special-type))))
(defn- sync-field-nested-fields! [driver field]
(defn- sync-field-nested-fields! [{:keys [features active-nested-field-name->type], :as driver} field]
(when (and (= (:base_type field) :DictionaryField)
(supports? driver :nested-fields) ; if one of these is true
(satisfies? ISyncDriverFieldNestedFields driver)) ; the other should be :wink:
(let [nested-field-name->type (active-nested-field-name->type driver field)]
(contains? features :nested-fields))
(let [nested-field-name->type (active-nested-field-name->type field)]
;; fetch existing nested fields
(let [existing-nested-field-name->id (sel :many :field->id [Field :name], :table_id (:table_id field), :active true, :parent_id (:id field))]
......
test/metabase/api/database_test.clj
+ 7
− 2
View file @ 3d9b139b
......@@ -25,7 +25,12 @@
;; ## GET /api/database/form_input
(expect
{:engines driver/available-drivers
{:engines (into {} (for [[driver info] @driver/available-drivers]
{driver (-> info
(update :details-fields (partial map (fn [field]
(cond-> field
(:type field) (update :type name)))))
(update :features (partial map name)))}))
:timezones ["GMT"
"UTC"
"US/Alaska"
......@@ -36,7 +41,7 @@
"US/Mountain"
"US/Pacific"
"America/Costa_Rica"]}
((user->client :crowberto) :get 200 "database/form_input"))
((user->client :crowberto) :get 200 "database/form_input"))
;; # DB LIFECYCLE ENDPOINTS
......
test/metabase/driver/generic_sql_test.clj
+ 5
− 7
View file @ 3d9b139b
(ns metabase.driver.generic-sql-test
(:require [expectations :refer :all]
[metabase.db :refer :all]
[metabase.driver :as driver]
(metabase.driver [h2 :as h2]
[interface :as i])
[metabase.driver.h2 :refer [h2]]
[metabase.driver.generic-sql.util :refer [korma-entity]]
(metabase.models [field :refer [Field]]
[foreign-key :refer [ForeignKey]]
......@@ -26,7 +24,7 @@
;; ACTIVE-TABLE-NAMES
(expect
#{"CATEGORIES" "VENUES" "CHECKINS" "USERS"}
(i/active-table-names h2/driver (db)))
((:active-table-names h2) (db)))
;; ACTIVE-COLUMN-NAMES->TYPE
(expect
......@@ -36,15 +34,15 @@
"PRICE" :IntegerField
"CATEGORY_ID" :IntegerField
"ID" :BigIntegerField}
(i/active-column-names->type h2/driver @venues-table))
((:active-column-names->type h2) @venues-table))
;; ## TEST TABLE-PK-NAMES
;; Pretty straightforward
(expect #{"ID"}
(i/table-pks h2/driver @venues-table))
((:table-pks h2) @venues-table))
;; ## TEST FIELD-AVG-LENGTH
(expect 13
(i/field-avg-length h2/driver @users-name-field))
((:field-avg-length h2) @users-name-field))
test/metabase/driver/h2_test.clj
+ 3
− 13
View file @ 3d9b139b
(ns metabase.driver.h2-test
(:require [expectations :refer :all]
[metabase.db :as db]
(metabase.driver [h2 :refer :all]
[interface :refer [can-connect?]])
[metabase.driver.generic-sql.interface :as i]
[metabase.driver.h2 :refer :all]
[metabase.test.util :refer [resolve-private-fns]]))
(resolve-private-fns metabase.driver.h2 connection-string->file+options file+options->connection-string connection-string-set-safe-options)
;; # Check that database->connection-details works
(expect {:db
"file:/Users/cam/birdly/bird_sightings.db;AUTO_SERVER=TRUE;DB_CLOSE_DELAY=-1"}
(i/database->connection-details driver {:details {:db "file:/Users/cam/birdly/bird_sightings.db;AUTO_SERVER=TRUE;DB_CLOSE_DELAY=-1"}}))
;; Check that the functions for exploding a connection string's options work as expected
(expect
["file:my-file" {"OPTION_1" "TRUE", "OPTION_2" "100", "LOOK_I_INCLUDED_AN_EXTRA_SEMICOLON" "NICE_TRY"}]
......@@ -34,8 +26,7 @@
;; Make sure we *cannot* connect to a non-existent database
(expect :exception-thrown
(try (can-connect? driver {:engine :h2
:details {:db (str (System/getProperty "user.dir") "/toucan_sightings")}})
(try ((:can-connect? h2) {:db (str (System/getProperty "user.dir") "/toucan_sightings")})
(catch org.h2.jdbc.JdbcSQLException e
(and (re-matches #"Database .+ not found .+" (.getMessage e))
:exception-thrown))))
......@@ -43,5 +34,4 @@
;; Check that we can connect to a non-existent Database when we enable potentailly unsafe connections (e.g. to the Metabase database)
(expect true
(binding [db/*allow-potentailly-unsafe-connections* true]
(can-connect? driver {:engine :h2
:details {:db (str (System/getProperty "user.dir") "/pigeon_sightings")}})))
((:can-connect? h2) {:db (str (System/getProperty "user.dir") "/pigeon_sightings")})))
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment