Skip to content
Snippets Groups Projects
Commit 19a3429b authored by Allen Gilliland's avatar Allen Gilliland
Browse files

Merge branch 'master' into fix_migration_for_mysql

parents 52607281 94edd687
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Showing
with 275 additions and 213 deletions
docs/developers-guide.md
+ 1
− 1
View file @ 19a3429b
......@@ -189,7 +189,7 @@ Will give you a list of out-of-date dependencies.
Once's this repo is made public, this Clojars badge will work and show the status as well:
[![Dependencies Status](http://jarkeeper.com/metabase/metabase-init/status.png)](http://jarkeeper.com/metabase/metabase)
[![Dependencies Status](https://jarkeeper.com/metabase/metabase-init/status.svg)](https://jarkeeper.com/metabase/metabase)
## Documentation
......
resources/migrations/019_add_schema_column_to_table.yaml 0 → 100644
+ 11
− 0
View file @ 19a3429b
databaseChangeLog:
- changeSet:
id: 19
author: camsaul
changes:
- addColumn:
tableName: metabase_table
columns:
- column:
name: schema
type: VARCHAR(256)
resources/migrations/liquibase.json
+ 2
− 1
View file @ 19a3429b
......@@ -16,6 +16,7 @@
{"include": {"file": "migrations/015_add_revision_is_creation_field.yaml"}},
{"include": {"file": "migrations/016_user_last_login_allow_null.yaml"}},
{"include": {"file": "migrations/017_add_database_is_sample_field.yaml"}},
{"include": {"file": "migrations/018_add_data_migrations_table.yaml"}}
{"include": {"file": "migrations/018_add_data_migrations_table.yaml"}},
{"include": {"file": "migrations/019_add_schema_column_to_table.yaml"}}
]
}
src/metabase/core.clj
+ 1
− 1
View file @ 19a3429b
......@@ -213,7 +213,7 @@
(defn- run-cmd [cmd & args]
(let [cmd->fn {:migrate (fn [direction]
(db/migrate (keyword direction)))}]
(db/migrate @db/db-connection-details (keyword direction)))}]
(if-let [f (cmd->fn cmd)]
(do (apply f args)
(println "Success.")
......
src/metabase/db.clj
+ 35
− 34
View file @ 19a3429b
......@@ -54,7 +54,7 @@
codec/form-decode
walk/keywordize-keys))))
(def ^:private db-connection-details
(def db-connection-details
"Connection details that can be used when pretending the Metabase DB is itself a `Database`
(e.g., to use the Generic SQL driver functions on the Metabase DB itself)."
(delay (or (when-let [uri (config/config-str :mb-db-connection-uri)]
......@@ -75,14 +75,16 @@
:user (config/config-str :mb-db-user)
:password (config/config-str :mb-db-pass)}))))
(def ^:private jdbc-connection-details
"Connection details for Korma / JDBC."
(delay (let [details @db-connection-details]
(case (:type details)
:h2 (kdb/h2 (assoc details :naming {:keys s/lower-case
:fields s/upper-case}))
:mysql (kdb/mysql (assoc details :db (:dbname details)))
:postgres (kdb/postgres (assoc details :db (:dbname details)))))))
(defn jdbc-details
"Takes our own MB details map and formats them properly for connection details for Korma / JDBC."
[db-details]
{:pre [(map? db-details)]}
;; TODO: it's probably a good idea to put some more validation here and be really strict about what's in `db-details`
(case (:type db-details)
:h2 (kdb/h2 (assoc db-details :naming {:keys s/lower-case
:fields s/upper-case}))
:mysql (kdb/mysql (assoc db-details :db (:dbname db-details)))
:postgres (kdb/postgres (assoc db-details :db (:dbname db-details)))))
;; ## MIGRATE
......@@ -100,24 +102,22 @@
* `:release-locks` - Manually release migration locks left by an earlier failed migration.
(This shouldn't be necessary now that we run migrations inside a transaction,
but is available just in case)."
([direction]
(migrate @jdbc-connection-details direction))
([jdbc-connection-details direction]
(try
(jdbc/with-db-transaction [conn jdbc-connection-details]
(let [^Database database (-> (DatabaseFactory/getInstance)
(.findCorrectDatabaseImplementation (JdbcConnection. (jdbc/get-connection conn))))
^Liquibase liquibase (Liquibase. changelog-file (ClassLoaderResourceAccessor.) database)]
(case direction
:up (.update liquibase "")
:down (.rollback liquibase 10000 "")
:down-one (.rollback liquibase 1 "")
:print (let [writer (StringWriter.)]
(.update liquibase "" writer)
(.toString writer))
:release-locks (.forceReleaseLocks liquibase))))
(catch Throwable e
(throw (DatabaseException. e))))))
[db-details direction]
(try
(jdbc/with-db-transaction [conn (jdbc-details db-details)]
(let [^Database database (-> (DatabaseFactory/getInstance)
(.findCorrectDatabaseImplementation (JdbcConnection. (jdbc/get-connection conn))))
^Liquibase liquibase (Liquibase. changelog-file (ClassLoaderResourceAccessor.) database)]
(case direction
:up (.update liquibase "")
:down (.rollback liquibase 10000 "")
:down-one (.rollback liquibase 1 "")
:print (let [writer (StringWriter.)]
(.update liquibase "" writer)
(.toString writer))
:release-locks (.forceReleaseLocks liquibase))))
(catch Throwable e
(throw (DatabaseException. e)))))
;; ## SETUP-DB
......@@ -148,23 +148,24 @@
(defn setup-db
"Do general perparation of database by validating that we can connect.
Caller can specify if we should run any pending database migrations."
[& {:keys [auto-migrate]
:or {auto-migrate true}}]
[& {:keys [db-details auto-migrate]
:or {db-details @db-connection-details
auto-migrate true}}]
(reset! setup-db-has-been-called? true)
;; Test DB connection and throw exception if we have any troubles connecting
(log/info "Verifying Database Connection ...")
(assert (db-can-connect? {:engine (:type @db-connection-details)
:details @db-connection-details})
(assert (db-can-connect? {:engine (:type db-details)
:details db-details})
"Unable to connect to Metabase DB.")
(log/info "Verify Database Connection ... CHECK")
;; Run through our DB migration process and make sure DB is fully prepared
(if auto-migrate
(migrate :up)
(migrate db-details :up)
;; if we are not doing auto migrations then print out migration sql for user to run manually
;; then throw an exception to short circuit the setup process and make it clear we can't proceed
(let [sql (migrate :print)]
(let [sql (migrate db-details :print)]
(log/info (str "Database Upgrade Required\n\n"
"NOTICE: Your database requires updates to work with this version of Metabase. "
"Please execute the following sql commands on your database before proceeding.\n\n"
......@@ -175,7 +176,7 @@
(log/info "Database Migrations Current ... CHECK")
;; Establish our 'default' Korma DB Connection
(kdb/default-connection (kdb/create-db @jdbc-connection-details))
(kdb/default-connection (kdb/create-db (jdbc-details db-details)))
;; Do any custom code-based migrations now that the db structure is up to date
;; NOTE: we use dynamic resolution to prevent circular dependencies
......
src/metabase/db/migrations.clj
+ 14
− 0
View file @ 19a3429b
......@@ -5,6 +5,7 @@
[metabase.db :as db]
(metabase.models [card :refer [Card]]
[database :refer [Database]]
[table :refer [Table]]
[setting :as setting])
[metabase.sample-data :as sample-data]
[metabase.util :as u]))
......@@ -66,3 +67,16 @@
(defmigration set-mongodb-databases-ssl-false
(doseq [{:keys [id details]} (db/sel :many :fields [Database :id :details] :engine "mongo")]
(db/upd Database id, :details (assoc details :ssl false))))
;; Set default values for :schema in existing tables now that we've added the column
;; That way sync won't get confused next time around
(defmigration set-default-schemas
(doseq [[engine default-schema] [["postgres" "public"]
["h2" "PUBLIC"]]]
(k/update Table
(k/set-fields {:schema default-schema})
(k/where {:schema nil
:db_id [in (k/subselect Database
(k/fields :id)
(k/where {:engine engine}))]}))))
src/metabase/driver.clj
+ 5
− 3
View file @ 19a3429b
......@@ -44,7 +44,7 @@
(def ^:private ^:const required-fns
#{:can-connect?
:active-table-names
:active-tables
:active-column-names->type
:table-pks
:field-values-lazy-seq
......@@ -171,9 +171,11 @@
Check whether we can connect to a `Database` with DETAILS-MAP and perform a simple query. For example, a SQL database might
try running a query like `SELECT 1;`. This function should return `true` or `false`.
* `(active-table-names [database])`
* `(active-tables [database])`
Return a set of string names of tables, collections, or equivalent that currently exist in DATABASE.
Return a set of maps containing information about the active tables/views, collections, or equivalent that currently exist in DATABASE.
Each map should contain the key `:name`, which is the string name of the table. For databases that have a concept of schemas,
this map should also include the string name of the table's `:schema`.
* `(active-column-names->type [table])`
......
src/metabase/driver/generic_sql.clj
+ 17
− 16
View file @ 19a3429b
......@@ -31,23 +31,19 @@
(with-jdbc-metadata [_ database]
(do-sync-fn)))
(defn- active-table-names [database]
(defn- active-tables [excluded-schemas database]
(with-jdbc-metadata [^java.sql.DatabaseMetaData md database]
(->> (.getTables md nil nil nil (into-array String ["TABLE", "VIEW"]))
jdbc/result-set-seq
(map :table_name)
set)))
(set (for [table (filter #(not (contains? excluded-schemas (:table_schem %)))
(jdbc/result-set-seq (.getTables md nil nil nil (into-array String ["TABLE", "VIEW"]))))]
{:name (:table_name table)
:schema (:table_schem table)}))))
(defn- active-column-names->type [column->base-type table]
{:pre [(map? column->base-type)]}
(with-jdbc-metadata [^java.sql.DatabaseMetaData md @(:db table)]
(->> (.getColumns md nil nil (:name table) nil)
jdbc/result-set-seq
(filter #(not= (:table_schem %) "INFORMATION_SCHEMA")) ; filter out internal tables
(map (fn [{:keys [column_name type_name]}]
{column_name (or (column->base-type (keyword type_name))
:UnknownField)}))
(into {}))))
(into {} (for [{:keys [column_name type_name]} (jdbc/result-set-seq (.getColumns md nil (:schema table) (:name table) nil))]
{column_name (or (column->base-type (keyword type_name))
:UnknownField)}))))
(defn- table-pks [table]
(with-jdbc-metadata [^java.sql.DatabaseMetaData md @(:db table)]
......@@ -188,6 +184,10 @@
Return a korma form for a date relative to NOW(), e.g. on that would produce SQL like `(NOW() + INTERVAL '1 month')`.
* `excluded-schemas` *(OPTIONAL)*
Set of string names of schemas to skip syncing tables from.
* `qp-clause->handler` *(OPTIONAL)*
A map of query processor clause keywords to functions of the form `(fn [korma-query query-map])` that are used apply them.
......@@ -198,14 +198,15 @@
;; Verify the driver
(verify-sql-driver driver)
(merge
{:features #{:foreign-keys
:standard-deviation-aggregations
:unix-timestamp-special-type-fields}
{:features (set (cond-> [:foreign-keys
:standard-deviation-aggregations
:unix-timestamp-special-type-fields]
(:set-timezone-sql driver) (conj :set-timezone)))
:qp-clause->handler qp/clause->handler
:can-connect? (partial can-connect? (:connection-details->spec driver))
:process-query process-query
:sync-in-context sync-in-context
:active-table-names active-table-names
:active-tables (partial active-tables (:excluded-schemas driver))
:active-column-names->type (partial active-column-names->type (:column->base-type driver))
:table-pks table-pks
:field-values-lazy-seq field-values-lazy-seq
......
src/metabase/driver/generic_sql/native.clj
+ 22
− 18
View file @ 19a3429b
......@@ -25,26 +25,30 @@
{:keys [features set-timezone-sql]} (driver/engine->driver (:engine database))]
(jdbc/with-db-transaction [t-conn db-conn]
(let [^java.sql.Connection jdbc-connection (:connection t-conn)]
;; Disable auto-commit for this transaction, that way shady queries are unable to modify the database
(.setAutoCommit jdbc-connection false)
(try
;; Set the timezone if applicable
(when-let [timezone (driver/report-timezone)]
(when (and (seq timezone)
(contains? features :set-timezone))
(log/debug (u/format-color 'green "%s" set-timezone-sql))
(try (jdbc/db-do-prepared t-conn set-timezone-sql [timezone])
(catch Throwable e
(log/error (u/format-color 'red "Failed to set timezone: %s" (.getMessage e)))))))
;; Set the timezone if applicable. We do this *before* making the transaction read-only because some DBs
;; won't let you set the timezone on a read-only connection
(when-let [timezone (driver/report-timezone)]
(when (and (seq timezone)
(contains? features :set-timezone))
(log/debug (u/format-color 'green "%s" set-timezone-sql))
(try (jdbc/db-do-prepared t-conn set-timezone-sql [timezone])
(catch Throwable e
(log/error (u/format-color 'red "Failed to set timezone: %s" (.getMessage e)))))))
;; Now run the query itself
(log/debug (u/format-color 'green "%s" sql))
(let [[columns & [first-row :as rows]] (jdbc/query t-conn sql, :as-arrays? true)]
{:rows rows
:columns columns
:cols (for [[column first-value] (zipmap columns first-row)]
{:name column
:base_type (value->base-type first-value)})})
;; Now make the transaction read-only and run the query itself
(.setReadOnly ^com.mchange.v2.c3p0.impl.NewProxyConnection (:connection t-conn) true)
(log/debug (u/format-color 'green "%s" sql))
(let [[columns & [first-row :as rows]] (jdbc/query t-conn sql, :as-arrays? true)]
{:rows rows
:columns columns
:cols (for [[column first-value] (zipmap columns first-row)]
{:name column
:base_type (value->base-type first-value)})})))
;; Rollback any changes made during this transaction just to be extra-double-sure JDBC doesn't try to commit them automatically for us
(finally (.rollback jdbc-connection))))))
(catch java.sql.SQLException e
(let [^String message (or (->> (.getMessage e) ; error message comes back like 'Column "ZID" not found; SQL statement: ... [error-code]' sometimes
(re-find #"^(.*);") ; the user already knows the SQL, and error code is meaningless
......
src/metabase/driver/generic_sql/query_processor.clj
+ 16
− 13
View file @ 19a3429b
(ns metabase.driver.generic-sql.query-processor
"The Query Processor is responsible for translating the Metabase Query Language into korma SQL forms."
(:require [clojure.core.match :refer [match]]
[clojure.tools.logging :as log]
[clojure.java.jdbc :as jdbc]
[clojure.string :as s]
[clojure.tools.logging :as log]
(korma [core :as k]
[db :as kdb])
(korma.sql [fns :as kfns]
[utils :as utils])
[metabase.config :as config]
[metabase.driver :as driver]
[metabase.driver.query-processor :as qp]
(metabase.driver.generic-sql [native :as native]
[util :refer :all])
[metabase.driver.query-processor :as qp]
[metabase.util :as u])
(:import java.sql.Timestamp
java.util.Date
......@@ -34,9 +35,9 @@
(formatted
([this]
(formatted this false))
([{:keys [table-name special-type field-name], :as field} include-as?]
([{:keys [schema-name table-name special-type field-name], :as field} include-as?]
(let [->timestamp (:unix-timestamp->timestamp (:driver *query*))
field (cond-> (keyword (str table-name \. field-name))
field (cond-> (keyword (str (when schema-name (str schema-name \.)) table-name \. field-name))
(= special-type :timestamp_seconds) (->timestamp :seconds)
(= special-type :timestamp_milliseconds) (->timestamp :milliseconds))]
(if include-as? [field (keyword field-name)]
......@@ -181,14 +182,15 @@
(defn- apply-page [korma-query {{:keys [items page]} :page}]
(-> korma-query
(k/limit items)
(k/offset (* items (dec page)))))
((-> *query* :driver :qp-clause->handler :limit) {:limit items}) ; lookup apply-limit from the driver and use that rather than calling k/limit directly
(k/offset (* items (dec page))))) ; so drivers that override it (like SQL Server) don't need to override this function as well
(defn- log-korma-form
[korma-form]
(when (config/config-bool :mb-db-logging)
(when-not qp/*disable-qp-logging*
(log/debug
(u/format-color 'green "\nKORMA FORM: 😋\n%s" (u/pprint-to-str (dissoc korma-form :db :ent :from :options :aliases :results :type :alias)))
(u/format-color 'blue "\nSQL: 😈\n%s\n" (-> (k/as-sql korma-form)
(s/replace #"\sFROM" "\nFROM") ; add newlines to the SQL to make it more readable
(s/replace #"\sLEFT JOIN" "\nLEFT JOIN")
......@@ -235,17 +237,18 @@
(kdb/with-db (:db entity)
(if (and (seq timezone)
(contains? (:features driver) :set-timezone))
(kdb/transaction
(try (k/exec-raw [(:set-timezone-sql driver) [timezone]])
(catch Throwable e
(log/error (u/format-color 'red "Failed to set timezone: %s" (.getMessage e)))))
(k/exec korma-query))
(try (kdb/transaction (k/exec-raw [(:set-timezone-sql driver) [timezone]])
(k/exec korma-query))
(catch Throwable e
(log/error (u/format-color 'red "Failed to set timezone:\n%s"
(with-out-str (jdbc/print-sql-exception-chain e))))
(k/exec korma-query)))
(k/exec korma-query))))
(catch java.sql.SQLException e
(let [^String message (or (->> (.getMessage e) ; error message comes back like "Error message ... [status-code]" sometimes
(let [^String message (or (->> (.getMessage e) ; error message comes back like "Error message ... [status-code]" sometimes
(re-find #"(?s)(^.*)\s+\[[\d-]+\]$") ; status code isn't useful and makes unit tests hard to write so strip it off
second) ; (?s) = Pattern.DOTALL - tell regex `.` to match newline characters as well
second) ; (?s) = Pattern.DOTALL - tell regex `.` to match newline characters as well
(.getMessage e))]
(throw (Exception. message)))))))
......
src/metabase/driver/generic_sql/util.clj
+ 4
− 2
View file @ 19a3429b
......@@ -76,9 +76,11 @@
([{db-delay :db, :as table}]
{:pre [(delay? db-delay)]}
(korma-entity @db-delay table))
([db {table-name :name}]
([db {schema :schema, table-name :name}]
{:pre [(map? db)]}
{:table table-name
{:table (if (seq schema)
(str schema \. table-name)
table-name)
:pk :id
:db (db->korma-db db)}))
......
src/metabase/driver/mongo.clj
+ 4
− 4
View file @ 19a3429b
......@@ -76,10 +76,10 @@
(with-mongo-connection [_ database]
(do-sync-fn)))
(defn- active-table-names [database]
(defn- active-tables [database]
(with-mongo-connection [^com.mongodb.DB conn database]
(-> (mdb/get-collection-names conn)
(set/difference #{"system.indexes"}))))
(set (for [collection (set/difference (mdb/get-collection-names conn) #{"system.indexes"})]
{:name collection}))))
(defn- active-column-names->type [table]
(with-mongo-connection [_ @(:db table)]
......@@ -147,7 +147,7 @@
:default false}]
:features #{:nested-fields}
:can-connect? can-connect?
:active-table-names active-table-names
:active-tables active-tables
:field-values-lazy-seq field-values-lazy-seq
:active-column-names->type active-column-names->type
:table-pks (constantly #{"_id"})
......
src/metabase/driver/mongo/util.clj
+ 7
− 8
View file @ 19a3429b
......@@ -44,13 +44,12 @@
"Run F with a new connection (bound to `*mongo-connection*`) to DATABASE.
Don't use this directly; use `with-mongo-connection`."
[f database]
;; The Mongo SSL detail is keyed by :use-ssl because the frontend has accidentally been saving all of the Mongo DBs with {:ssl true}
(let [{:keys [dbname host port user pass use-ssl]
:or {port 27017, pass "", use-ssl false}} (cond
(string? database) {:dbname database}
(:dbname (:details database)) (:details database) ; entire Database obj
(:dbname database) database ; connection details map only
:else (throw (Exception. (str "with-mongo-connection failed: bad connection details:" (:details database)))))
(let [{:keys [dbname host port user pass ssl]
:or {port 27017, pass "", ssl false}} (cond
(string? database) {:dbname database}
(:dbname (:details database)) (:details database) ; entire Database obj
(:dbname database) database ; connection details map only
:else (throw (Exception. (str "with-mongo-connection failed: bad connection details:" (:details database)))))
user (when (seq user) ; ignore empty :user and :pass strings
user)
pass (when (seq pass)
......@@ -58,7 +57,7 @@
server-address (mg/server-address host port)
credentials (when user
(mcred/create user dbname pass))
connect (partial mg/connect server-address (build-connection-options :ssl? use-ssl))
connect (partial mg/connect server-address (build-connection-options :ssl? ssl))
conn (if credentials
(connect credentials)
(connect))
......
src/metabase/driver/mysql.clj
+ 33
− 33
View file @ 19a3429b
......@@ -138,36 +138,36 @@
message))
(defdriver mysql
(-> {:driver-name "MySQL"
:details-fields [{:name "host"
:display-name "Host"
:default "localhost"}
{:name "port"
:display-name "Port"
:type :integer
:default 3306}
{:name "dbname"
:display-name "Database name"
:placeholder "birds_of_the_word"
:required true}
{:name "user"
:display-name "Database username"
:placeholder "What username do you use to login to the database?"
:required true}
{:name "password"
:display-name "Database password"
:type :password
:placeholder "*******"}]
:column->base-type column->base-type
:sql-string-length-fn :CHAR_LENGTH
:connection-details->spec connection-details->spec
:unix-timestamp->timestamp unix-timestamp->timestamp
:date date
:date-interval date-interval
;; If this fails you need to load the timezone definitions from your system into MySQL;
;; run the command `mysql_tzinfo_to_sql /usr/share/zoneinfo | mysql -u root mysql`
;; See https://dev.mysql.com/doc/refman/5.7/en/time-zone-support.html for details
:set-timezone-sql "SET @@session.time_zone = ?;"
:humanize-connection-error-message humanize-connection-error-message}
sql-driver
(update :features conj :set-timezone)))
(sql-driver
{:driver-name "MySQL"
:details-fields [{:name "host"
:display-name "Host"
:default "localhost"}
{:name "port"
:display-name "Port"
:type :integer
:default 3306}
{:name "dbname"
:display-name "Database name"
:placeholder "birds_of_the_word"
:required true}
{:name "user"
:display-name "Database username"
:placeholder "What username do you use to login to the database?"
:required true}
{:name "password"
:display-name "Database password"
:type :password
:placeholder "*******"}]
:column->base-type column->base-type
:sql-string-length-fn :CHAR_LENGTH
:excluded-schemas #{"INFORMATION_SCHEMA"}
:connection-details->spec connection-details->spec
:unix-timestamp->timestamp unix-timestamp->timestamp
:date date
:date-interval date-interval
;; If this fails you need to load the timezone definitions from your system into MySQL;
;; run the command `mysql_tzinfo_to_sql /usr/share/zoneinfo | mysql -u root mysql`
;; See https://dev.mysql.com/doc/refman/5.7/en/time-zone-support.html for details
:set-timezone-sql "SET @@session.time_zone = ?;"
:humanize-connection-error-message humanize-connection-error-message}))
src/metabase/driver/postgres.clj
+ 34
− 35
View file @ 19a3429b
......@@ -166,38 +166,37 @@
message))
(defdriver postgres
(-> {:driver-name "PostgreSQL"
:details-fields [{:name "host"
:display-name "Host"
:default "localhost"}
{:name "port"
:display-name "Port"
:type :integer
:default 5432}
{:name "dbname"
:display-name "Database name"
:placeholder "birds_of_the_word"
:required true}
{:name "user"
:display-name "Database username"
:placeholder "What username do you use to login to the database?"
:required true}
{:name "password"
:display-name "Database password"
:type :password
:placeholder "*******"}
{:name "ssl"
:display-name "Use a secure connection (SSL)?"
:type :boolean
:default false}]
:sql-string-length-fn :CHAR_LENGTH
:column->base-type column->base-type
:connection-details->spec connection-details->spec
:unix-timestamp->timestamp unix-timestamp->timestamp
:date date
:date-interval date-interval
:set-timezone-sql "SET LOCAL timezone TO ?;"
:driver-specific-sync-field! driver-specific-sync-field!
:humanize-connection-error-message humanize-connection-error-message}
sql-driver
(update :features conj :set-timezone)))
(sql-driver
{:driver-name "PostgreSQL"
:details-fields [{:name "host"
:display-name "Host"
:default "localhost"}
{:name "port"
:display-name "Port"
:type :integer
:default 5432}
{:name "dbname"
:display-name "Database name"
:placeholder "birds_of_the_word"
:required true}
{:name "user"
:display-name "Database username"
:placeholder "What username do you use to login to the database?"
:required true}
{:name "password"
:display-name "Database password"
:type :password
:placeholder "*******"}
{:name "ssl"
:display-name "Use a secure connection (SSL)?"
:type :boolean
:default false}]
:sql-string-length-fn :CHAR_LENGTH
:column->base-type column->base-type
:connection-details->spec connection-details->spec
:unix-timestamp->timestamp unix-timestamp->timestamp
:date date
:date-interval date-interval
:set-timezone-sql "UPDATE pg_settings SET setting = ? WHERE name ILIKE 'timezone';"
:driver-specific-sync-field! driver-specific-sync-field!
:humanize-connection-error-message humanize-connection-error-message}))
src/metabase/driver/query_processor/annotate.clj
+ 1
− 0
View file @ 19a3429b
......@@ -232,6 +232,7 @@
:field-id :id
:field-name :name
:field-display-name :display_name
:schema-name :schema_name
:special-type :special_type
:preview-display :preview_display
:table-id :table_id})
......
src/metabase/driver/query_processor/interface.clj
+ 1
− 0
View file @ 19a3429b
......@@ -40,6 +40,7 @@
^Keyword base-type
^Keyword special-type
^Integer table-id
^String schema-name
^String table-name
^Integer position
^String description
......
src/metabase/driver/query_processor/resolve.clj
+ 6
− 3
View file @ 19a3429b
......@@ -79,8 +79,11 @@
:else this))
(defmethod resolve-table Field [{:keys [table-id], :as this} table-id->table]
(assoc this :table-name (:name (or (table-id->table table-id)
(throw (Exception. (format "Query expansion failed: could not find table %d." table-id)))))))
(let [table (or (table-id->table table-id)
(throw (Exception. (format "Query expansion failed: could not find table %d." table-id))))]
(assoc this
:table-name (:name table)
:schema-name (:schema table))))
;; ## FieldPlaceholder
......@@ -194,7 +197,7 @@
[{{source-table-id :source-table} :query, database-id :database, :keys [table-ids fk-field-ids], :as expanded-query-dict}]
{:pre [(integer? source-table-id)]}
(let [table-ids (conj table-ids source-table-id)
table-id->table (sel :many :id->fields [Table :name :id] :id [in table-ids])
table-id->table (sel :many :id->fields [Table :schema :name :id] :id [in table-ids])
join-tables (vals (dissoc table-id->table source-table-id))]
(-<> expanded-query-dict
......
src/metabase/driver/sync.clj
+ 59
− 39
View file @ 19a3429b
......@@ -38,46 +38,66 @@
;; ## sync-database! and sync-table!
(defn- -sync-database! [{:keys [active-table-names], :as driver} database]
(let [start-time (System/currentTimeMillis)
(defn- validate-active-tables [results]
(when-not (and (set? results)
(every? map? results)
(every? :name results))
(throw (Exception. "Invalid results returned by active-tables. Results should be a set of maps like {:name \"table_name\", :schema \"schema_name_or_nil\"}."))))
(defn- mark-inactive-tables!
"Mark any `Tables` that are no longer active as such. These are ones that exist in the DB but didn't come back from `active-tables`."
[database active-tables existing-table->id]
(doseq [[[{table :name, schema :schema, :as table}] table-id] existing-table->id]
(when-not (contains? active-tables table)
(upd Table table-id :active false)
(log/info (u/format-color 'cyan "Marked table %s.%s%s as inactive." (:name database) (if schema (str schema \.) "") table))
;; We need to mark driver Table's Fields as inactive so we don't expose them in UI such as FK selector (etc.)
(k/update Field
(k/where {:table_id table-id})
(k/set-fields {:active false})))))
(defn- create-new-tables!
"Create new `Tables` as needed. These are ones that came back from `active-tables` but don't already exist in the DB."
[database active-tables existing-table->id]
(let [existing-tables (set (keys existing-table->id))
new-tables (set/difference active-tables existing-tables)]
(when (seq new-tables)
(log/debug (u/format-color 'blue "Found new tables: %s" (for [{table :name, schema :schema} new-tables]
(if schema
(str schema \. table)
table))))
(doseq [{table :name, schema :schema} new-tables]
;; If it's a _metabase_metadata table then we'll handle later once everything else has been synced
(when-not (= (s/lower-case table) "_metabase_metadata")
(ins Table :db_id (:id database), :active true, :schema schema, :name table))))))
(defn- fetch-and-sync-database-active-tables! [driver database]
(sync-database-active-tables! driver (for [table (sel :many Table, :db_id (:id database) :active true)]
;; replace default delays with ones that reuse database (and don't require a DB call)
(assoc table :db (delay database)))))
(defn- -sync-database! [{:keys [active-tables], :as driver} database]
(let [active-tables (active-tables database)
existing-table->id (into {} (for [{:keys [name schema id]} (sel :many :fields [Table :name :schema :id], :db_id (:id database), :active true)]
{{:name name, :schema schema} id}))]
(validate-active-tables active-tables)
(mark-inactive-tables! database active-tables existing-table->id)
(create-new-tables! database active-tables existing-table->id))
(fetch-and-sync-database-active-tables! driver database)
;; Ok, now if we had a _metabase_metadata table from earlier we can go ahead and sync from it
(sync-metabase-metadata-table! driver database))
(defn- -sync-database-with-tracking! [driver database]
(let [start-time (System/currentTimeMillis)
tracking-hash (str (java.util.UUID/randomUUID))]
(log/info (u/format-color 'magenta "Syncing %s database '%s'..." (name (:engine database)) (:name database)))
(events/publish-event :database-sync-begin {:database_id (:id database) :custom_id tracking-hash})
(let [active-table-names (active-table-names database)
table-name->id (sel :many :field->id [Table :name] :db_id (:id database) :active true)]
(assert (set? active-table-names) "active-table-names should return a set.")
(assert (every? string? active-table-names) "active-table-names should return the names of Tables as *strings*.")
;; First, let's mark any Tables that are no longer active as such.
;; These are ones that exist in table-name->id but not in active-table-names.
(doseq [[table-name table-id] table-name->id]
(when-not (contains? active-table-names table-name)
(upd Table table-id :active false)
(log/info (u/format-color 'cyan "Marked table %s.%s as inactive." (:name database) table-name))
;; We need to mark driver Table's Fields as inactive so we don't expose them in UI such as FK selector (etc.)
(k/update Field
(k/where {:table_id table-id})
(k/set-fields {:active false}))))
;; Next, we'll create new Tables (ones that came back in active-table-names but *not* in table-name->id)
(let [existing-table-names (set (keys table-name->id))
new-table-names (set/difference active-table-names existing-table-names)]
(when (seq new-table-names)
(log/debug (u/format-color 'blue "Found new tables: %s" new-table-names))
(doseq [new-table-name new-table-names]
;; If it's a _metabase_metadata table then we'll handle later once everything else has been synced
(when-not (= (s/lower-case new-table-name) "_metabase_metadata")
(ins Table :db_id (:id database), :active true, :name new-table-name))))))
;; Now sync the active tables
(->> (sel :many Table :db_id (:id database) :active true)
(map #(assoc % :db (delay database))) ; replace default delays with ones that reuse database (and don't require a DB call)
(sync-database-active-tables! driver))
;; Ok, now if we had a _metabase_metadata table from earlier we can go ahead and sync from it
(sync-metabase-metadata-table! driver database)
(-sync-database! driver database)
(events/publish-event :database-sync-end {:database_id (:id database) :custom_id tracking-hash :running_time (- (System/currentTimeMillis) start-time)})
(log/info (u/format-color 'magenta "Finished syncing %s database %s. (%d ms)" (name (:engine database)) (:name database)
......@@ -88,7 +108,7 @@
[{:keys [sync-in-context], :as driver} database]
(binding [qp/*disable-qp-logging* true
*sel-disable-logging* true]
(let [f (partial -sync-database! driver database)]
(let [f (partial -sync-database-with-tracking! driver database)]
(if sync-in-context
(sync-in-context database f)
(f)))))
......@@ -108,9 +128,9 @@
`keypath` is of the form `table-name.key` or `table-name.field-name.key`, where `key` is the name of some property of `Table` or `Field`.
This functionality is currently only used by the Sample Dataset. In order to use this functionality, drivers must implement optional fn `:table-rows-seq`."
[{:keys [table-rows-seq active-table-names]} database]
[{:keys [table-rows-seq active-tables]} database]
(when table-rows-seq
(doseq [table-name (active-table-names database)]
(doseq [{table-name :name} (active-tables database)]
(when (= (s/lower-case table-name) "_metabase_metadata")
(doseq [{:keys [keypath value]} (table-rows-seq database table-name)]
(let [[_ table-name field-name k] (re-matches #"^([^.]+)\.(?:([^.]+)\.)?([^.]+)$" keypath)]
......
src/metabase/util.clj
+ 2
− 2
View file @ 19a3429b
......@@ -228,9 +228,9 @@
`(try
(~f ~@params)
(catch Throwable e#
(log/error (color/red ~(format "Caught exception in %s:" f)
(log/error (color/red ~(format "Caught exception in %s: " f)
(or (.getMessage e#) e#)
(with-out-str (.printStackTrace e#)))))))
#_(with-out-str (.printStackTrace e#)))))))
(defn indecies-satisfying
"Return a set of indencies in COLL that satisfy PRED.
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment