Skip to content
Snippets Groups Projects
Commit 38a827cc authored by Cam Saül's avatar Cam Saül Committed by Cam Saül
Browse files

Redshift driver :yum:

parent a3d5cc94
No related branches found
No related tags found
No related merge requests found
......@@ -25,11 +25,11 @@ test:
# 0) runs unit tests w/ H2 local DB. Runs against H2, Mongo, MySQL
# 1) runs unit tests w/ Postgres local DB. Runs against H2, SQL Server
# 2) runs unit tests w/ MySQL local DB. Runs against H2, Postgres, SQLite
# 3) runs Eastwood linter
# 4) Bikeshed linter
# 3) runs unit tests w/ H2 local DB. Runs against H2, Redshift
# 4) runs Eastwood linter & Bikeshed linter
# 5) runs JS linter + JS test
# 6) runs lein uberjar. (We don't run bin/build because we're not really concerned about `npm install` (etc) in this test, which runs elsewhere)
- case $CIRCLE_NODE_INDEX in 0) ENGINES=h2,mongo,mysql lein test ;; 1) ENGINES=h2,sqlserver MB_DB_TYPE=postgres MB_DB_DBNAME=circle_test MB_DB_PORT=5432 MB_DB_USER=ubuntu MB_DB_HOST=localhost lein test ;; 2) ENGINES=h2,postgres,sqlite MB_DB_TYPE=mysql MB_DB_DBNAME=circle_test MB_DB_PORT=3306 MB_DB_USER=ubuntu MB_DB_HOST=localhost lein test ;; 3) lein eastwood ;; 4) lein bikeshed --max-line-length 240 ;; 5) npm install && npm run lint && npm run build && npm run test ;; 6) lein uberjar ;; esac:
- case $CIRCLE_NODE_INDEX in 0) ENGINES=h2,mongo,mysql lein test ;; 1) ENGINES=h2,sqlserver MB_DB_TYPE=postgres MB_DB_DBNAME=circle_test MB_DB_PORT=5432 MB_DB_USER=ubuntu MB_DB_HOST=localhost lein test ;; 2) ENGINES=h2,postgres,sqlite MB_DB_TYPE=mysql MB_DB_DBNAME=circle_test MB_DB_PORT=3306 MB_DB_USER=ubuntu MB_DB_HOST=localhost lein test ;; 3) ENGINES=h2,redshift lein test ;; 4) lein eastwood && lein bikeshed --max-line-length 240 ;; 5) npm install && npm run lint && npm run build && npm run test ;; 6) lein uberjar ;; esac:
parallel: true
deployment:
master:
......
......@@ -9,7 +9,7 @@
[metabase.driver :as driver]
[metabase.driver.query-processor :as qp]))
(defn- db->connection-spec
(defn db->connection-spec
"Return a JDBC connection spec for a Metabase `Database`."
[{{:keys [short-lived?]} :details, :as database}]
(let [driver (driver/engine->driver (:engine database))]
......
......@@ -78,7 +78,7 @@
(keyword "timestamp with timezone") :DateTimeField
(keyword "timestamp without timezone") :DateTimeField} column-type))
(def ^:private ^:const ssl-params
(def ^:const ssl-params
"Params to include in the JDBC connection spec for an SSL connection."
{:ssl true
:sslmode "require"
......@@ -162,13 +162,23 @@
clojure.lang.Named
(getName [_] "PostgreSQL"))
(def PostgresISQLDriverMixin
"Implementations of `ISQLDriver` methods for `PostgresDriver`."
(merge (sql/ISQLDriverDefaultsMixin)
{:column->base-type column->base-type
:connection-details->spec connection-details->spec
:date date
:set-timezone-sql (constantly "UPDATE pg_settings SET setting = ? WHERE name ILIKE 'timezone';")
:string-length-fn (constantly :CHAR_LENGTH)
:unix-timestamp->timestamp unix-timestamp->timestamp}))
(extend PostgresDriver
driver/IDriver
(merge (sql/IDriverSQLDefaultsMixin)
{:date-interval date-interval
:details-fields (constantly [{:name "host"
:display-name "Host"
:default "localhost"}
:default "localhost"}
{:name "port"
:display-name "Port"
:type :integer
......@@ -192,13 +202,6 @@
:driver-specific-sync-field! driver-specific-sync-field!
:humanize-connection-error-message humanize-connection-error-message})
sql/ISQLDriver
(merge (sql/ISQLDriverDefaultsMixin)
{:column->base-type column->base-type
:connection-details->spec connection-details->spec
:date date
:set-timezone-sql (constantly "UPDATE pg_settings SET setting = ? WHERE name ILIKE 'timezone';")
:string-length-fn (constantly :CHAR_LENGTH)
:unix-timestamp->timestamp unix-timestamp->timestamp}))
sql/ISQLDriver PostgresISQLDriverMixin)
(driver/register-driver! :postgres (PostgresDriver.))
(ns metabase.driver.redshift
"Amazon Redshift Driver."
(:require [clojure.java.jdbc :as jdbc]
(korma [core :as k]
[db :as kdb])
[korma.sql.utils :as kutils]
(metabase [config :as config]
[driver :as driver])
[metabase.driver.generic-sql :as sql]
[metabase.driver.generic-sql.util :as sqlutil]
[metabase.driver.postgres :as postgres]))
(defn- connection-details->spec [_ details]
(kdb/postgres (merge details postgres/ssl-params))) ; always connect to redshift over SSL
(defn- date-interval [_ unit amount]
(kutils/generated (format "(GETDATE() + INTERVAL '%d %s')" amount (name unit))))
(defn- unix-timestamp->timestamp [_ field-or-value seconds-or-milliseconds]
(kutils/func (case seconds-or-milliseconds
:seconds "(TIMESTAMP '1970-01-01T00:00:00Z' + (%s * INTERVAL '1 second'))"
:milliseconds "(TIMESTAMP '1970-01-01T00:00:00Z' + ((%s / 1000) * INTERVAL '1 second'))")
[field-or-value]))
;; The Postgres JDBC .getImportedKeys method doesn't work for Redshift, and we're not allowed to access information_schema.constraint_column_usage,
;; so we'll have to use this custome query instead
;; See also: [Related Postgres JDBC driver issue on GitHub](https://github.com/pgjdbc/pgjdbc/issues/79)
;; [How to access the equivalent of information_schema.constraint_column_usage in Redshift](https://forums.aws.amazon.com/thread.jspa?threadID=133514)
(defn- table-fks [_ table]
(set (jdbc/query (sqlutil/db->connection-spec @(:db table))
["SELECT source_column.attname AS \"fk-column-name\",
dest_table.relname AS \"dest-table-name\",
dest_column.attname AS \"dest-column-name\"
FROM pg_constraint c
JOIN pg_namespace n ON c.connamespace = n.oid
JOIN pg_class source_table ON c.conrelid = source_table.oid
JOIN pg_attribute source_column ON c.conrelid = source_column.attrelid
JOIN pg_class dest_table ON c.confrelid = dest_table.oid
JOIN pg_attribute dest_column ON c.confrelid = dest_column.attrelid
WHERE c.contype = 'f'::char
AND source_table.relname = ?
AND n.nspname = ?
AND source_column.attnum = ANY(c.conkey)
AND dest_column.attnum = ANY(c.confkey)"
(:name table)
(:schema table)])))
(defrecord RedshiftDriver []
clojure.lang.Named
(getName [_] "Amazon Redshift"))
(extend RedshiftDriver
driver/IDriver
(merge (sql/IDriverSQLDefaultsMixin)
{:date-interval date-interval
:details-fields (constantly [{:name "host"
:display-name "Host"
:placeholder "my-cluster-name.abcd1234.us-east-1.redshift.amazonaws.com"
:required true}
{:name "port"
:display-name "Port"
:type :integer
:default 5439}
{:name "db"
:display-name "Database name"
:placeholder "toucan_sightings"
:required true}
{:name "user"
:display-name "Master username"
:placeholder "cam"
:required true}
{:name "password"
:display-name "Master user password"
:type :password
:placeholder "*******"
:required true}])
:table-fks table-fks})
sql/ISQLDriver
(merge postgres/PostgresISQLDriverMixin
{:connection-details->spec connection-details->spec
:current-datetime-fn (constantly (k/sqlfn* :GETDATE))
:set-timezone-sql (constantly nil)
:unix-timestamp->timestamp unix-timestamp->timestamp}
;; HACK ! When we test against Redshift we use a session-unique schema so we can run simultaneous tests against a single remote host;
;; when running tests tell the sync process to ignore all the other schemas
(when config/is-test?
{:excluded-schemas (memoize
(fn [_]
(require 'metabase.test.data.redshift)
(let [session-schema-number @(resolve 'metabase.test.data.redshift/session-schema-number)]
(set (conj (for [i (range 240)
:when (not= i session-schema-number)]
(str "schema_" i))
"public")))))})))
(driver/register-driver! :redshift (RedshiftDriver.))
......@@ -30,7 +30,7 @@
(set/difference datasets/all-valid-engines (engines-that-support feature)))
(defmacro if-questionable-timezone-support [then else]
`(if (contains? #{:sqlserver :mongo :sqlite} *engine*)
`(if (contains? #{:mongo :redshift :sqlite :sqlserver} *engine*)
~then
~else))
......@@ -269,7 +269,9 @@
;; ## "AVG" AGGREGATION
(qp-expect-with-all-engines
{:rows [[35.50589199999998]]
{:rows [[(if (= *engine* :redshift)
35.505892
35.50589199999998)]]
:columns ["avg"]
:cols [(aggregate-col :avg (venues-col :latitude))]}
(Q aggregate avg latitude of venues))
......@@ -389,7 +391,8 @@
:columns (venues-columns)
:cols (venues-cols)}
(Q aggregate rows of venues
filter between id 21 22))
filter between id 21 22
order id+))
;; ### FILTER -- "BETWEEN" with dates
(qp-expect-with-all-engines
......@@ -408,7 +411,8 @@
:columns (venues-columns)
:cols (venues-cols)}
(Q aggregate rows of venues
filter or <= id 3 = id 5))
filter or <= id 3 = id 5
order id+))
;; ### FILTER -- "INSIDE"
(qp-expect-with-all-engines
......@@ -596,13 +600,14 @@
:cols [(aggregate-col :stddev (venues-col :latitude))]
:rows [[(datasets/engine-case
:h2 3.43467255295115 ; annoying :/
:postgres 3.4346725529512736
:mysql 3.417456040761316
:postgres 3.4346725529512736
:redshift 3.43467255295115
:sqlserver 3.43467255295126)]]}
(Q aggregate stddev latitude of venues))
;; Make sure standard deviation fails for the Mongo driver since its not supported
(datasets/expect-with-engines (engines-that-dont-support :foreign-keys)
(datasets/expect-with-engines (engines-that-dont-support :standard-deviation-aggregations)
{:status :failed
:error "standard-deviation-aggregations is not supported by this driver."}
(select-keys (Q aggregate stddev latitude of venues) [:status :error]))
......@@ -661,32 +666,36 @@
"avg"]
:rows [[3 (datasets/engine-case
:h2 22
:postgres 22.0000000000000000M
:mysql 22.0000M
:sqlserver 22
:mongo 22.0
:sqlite 22.0)]
:mysql 22.0000M
:postgres 22.0000000000000000M
:redshift 22
:sqlite 22.0
:sqlserver 22)]
[2 (datasets/engine-case
:h2 28
:postgres 28.2881355932203390M
:mysql 28.2881M
:sqlserver 28
:mongo 28.28813559322034
:sqlite 28.28813559322034)]
:mysql 28.2881M
:postgres 28.2881355932203390M
:redshift 28
:sqlite 28.28813559322034
:sqlserver 28)]
[1 (datasets/engine-case
:h2 32
:postgres 32.8181818181818182M
:mysql 32.8182M
:sqlserver 32
:mongo 32.81818181818182
:sqlite 32.81818181818182)]
:mysql 32.8182M
:postgres 32.8181818181818182M
:redshift 32
:sqlite 32.81818181818182
:sqlserver 32)]
[4 (datasets/engine-case
:h2 53
:postgres 53.5000000000000000M
:mysql 53.5000M
:sqlserver 53
:mongo 53.5
:sqlite 53.5)]]
:mysql 53.5000M
:postgres 53.5000000000000000M
:redshift 53
:sqlite 53.5
:sqlserver 53)]]
:cols [(venues-col :price)
(aggregate-col :avg (venues-col :category_id))]}
(Q return :data
......@@ -701,10 +710,10 @@
(datasets/expect-with-engines (engines-that-support :standard-deviation-aggregations)
{:columns [(format-name "price")
"stddev"]
:rows [[3 (datasets/engine-case :h2 26, :postgres 26, :mysql 25, :sqlserver 26)]
:rows [[3 (datasets/engine-case :h2 26, :mysql 25, :postgres 26, :redshift 26, :sqlserver 26)]
[1 24]
[2 21]
[4 (datasets/engine-case :h2 15, :postgres 15, :mysql 14, :sqlserver 15)]]
[4 (datasets/engine-case :h2 15, :mysql 14, :postgres 15, :redshift 15, :sqlserver 15)]]
:cols [(venues-col :price)
(aggregate-col :stddev (venues-col :category_id))]}
(-> (Q return :data
......@@ -784,8 +793,8 @@
(datasets/expect-with-all-engines
(cond
;; SQL Server and Mongo don't have a concept of timezone so results are all grouped by UTC
(contains? #{:sqlserver :mongo} *engine*)
;; SQL Server, Mongo, and Redshift don't have a concept of timezone so results are all grouped by UTC
(contains? #{:mongo :redshift :sqlserver} *engine*)
[[#inst "2015-06-01T07" 6]
[#inst "2015-06-02T07" 10]
[#inst "2015-06-03T07" 4]
......@@ -809,7 +818,7 @@
["2015-06-09" 7]
["2015-06-10" 9]]
;; Postgres, MySQL, and H2 -- grouped by DB timezone, US/Pacific in this case
;; Postgres, Redshift, MySQL, and H2 -- grouped by DB timezone, US/Pacific in this case
:else
[[#inst "2015-06-01T07" 8]
[#inst "2015-06-02T07" 9]
......@@ -1134,7 +1143,7 @@
(datasets/expect-with-all-engines
(cond
(= *engine* :sqlserver)
(contains? #{:redshift :sqlserver} *engine*)
[[#inst "2015-06-01T17:31" 1]
[#inst "2015-06-01T23:06" 1]
[#inst "2015-06-02T00:23" 1]
......@@ -1173,7 +1182,7 @@
(datasets/expect-with-all-engines
(cond
(contains? #{:sqlserver :mongo} *engine*)
(contains? #{:mongo :redshift :sqlserver} *engine*)
[[#inst "2015-06-01T17:31" 1]
[#inst "2015-06-01T23:06" 1]
[#inst "2015-06-02T00:23" 1]
......@@ -1225,7 +1234,7 @@
(datasets/expect-with-all-engines
(cond
(contains? #{:sqlserver :mongo} *engine*)
(contains? #{:mongo :redshift :sqlserver} *engine*)
[[#inst "2015-06-01T17" 1]
[#inst "2015-06-01T23" 1]
[#inst "2015-06-02T00" 1]
......@@ -1270,7 +1279,7 @@
(datasets/expect-with-all-engines
(cond
(contains? #{:sqlserver :mongo} *engine*)
(contains? #{:mongo :redshift :sqlserver} *engine*)
[[#inst "2015-06-01T07" 6]
[#inst "2015-06-02T07" 10]
[#inst "2015-06-03T07" 4]
......@@ -1327,7 +1336,7 @@
(datasets/expect-with-all-engines
(cond
(contains? #{:sqlserver :mongo} *engine*)
(contains? #{:mongo :redshift :sqlserver} *engine*)
[[#inst "2015-05-31T07" 46]
[#inst "2015-06-07T07" 47]
[#inst "2015-06-14T07" 40]
......@@ -1354,7 +1363,7 @@
(contains? #{:sqlserver :sqlite} *engine*)
[[23 54] [24 46] [25 39] [26 61]]
(= *engine* :mongo)
(contains? #{:mongo :redshift} *engine*)
[[23 46] [24 47] [25 40] [26 60] [27 7]]
:else
......
......@@ -11,6 +11,7 @@
[mongo :refer [map->MongoDriver]]
[mysql :refer [map->MySQLDriver]]
[postgres :refer [map->PostgresDriver]]
[redshift :refer [map->RedshiftDriver]]
[sqlite :refer [map->SQLiteDriver]]
[sqlserver :refer [map->SQLServerDriver]])
(metabase.models [field :refer [Field]]
......@@ -20,6 +21,7 @@
[mongo :as mongo]
[mysql :as mysql]
[postgres :as postgres]
[redshift :as redshift]
[sqlite :as sqlite]
[sqlserver :as sqlserver])
[metabase.util :as u])
......@@ -27,6 +29,7 @@
metabase.driver.mongo.MongoDriver
metabase.driver.mysql.MySQLDriver
metabase.driver.postgres.PostgresDriver
metabase.driver.redshift.RedshiftDriver
metabase.driver.sqlite.SQLiteDriver
metabase.driver.sqlserver.SQLServerDriver))
......@@ -76,7 +79,7 @@
:timestamp-field-type (constantly :DateField)}))
;; ## Generic SQL
;; ## SQL Drivers
(def ^:private GenericSQLIDatasetMixin
(merge IDatasetDefaultsMixin
......@@ -85,8 +88,6 @@
:timestamp-field-type (constantly :DateTimeField)}))
;;; ### H2
(extend H2Driver
IDataset
(merge GenericSQLIDatasetMixin
......@@ -97,7 +98,11 @@
:sum-field-type (constantly :BigIntegerField)}))
;;; ### Postgres
(extend MySQLDriver
IDataset
(merge GenericSQLIDatasetMixin
{:sum-field-type (constantly :BigIntegerField)}))
(extend PostgresDriver
IDataset
......@@ -105,23 +110,17 @@
{:default-schema (constantly "public")}))
;;; ### MySQL
(extend MySQLDriver
(extend RedshiftDriver
IDataset
(merge GenericSQLIDatasetMixin
{:sum-field-type (constantly :BigIntegerField)}))
{:default-schema (constantly redshift/session-schema-name)}))
;;; ### SQLite
(extend SQLiteDriver
IDataset
GenericSQLIDatasetMixin)
;;; ### SQLServer
(extend SQLServerDriver
IDataset
(merge GenericSQLIDatasetMixin
......@@ -137,6 +136,7 @@
:mongo (map->MongoDriver {:dbpromise (promise)})
:mysql (map->MySQLDriver {:dbpromise (promise)})
:postgres (map->PostgresDriver {:dbpromise (promise)})
:redshift (map->RedshiftDriver {:dbpromise (promise)})
:sqlite (map->SQLiteDriver {:dbpromise (promise)})
:sqlserver (map->SQLServerDriver {:dbpromise (promise)})})
......
......@@ -173,7 +173,9 @@
(defn default-execute-sql! [loader context dbdef sql]
(let [sql (some-> sql s/trim)]
(when (seq sql)
(when (and (seq sql)
;; make sure SQL isn't just semicolons
(not (s/blank? (s/replace sql #";" ""))))
(try
(jdbc/execute! (database->spec loader context dbdef) [sql] :transaction? false, :multi? true)
(catch java.sql.SQLException e
......
(ns metabase.test.data.redshift
(:require [clojure.java.jdbc :as jdbc]
[clojure.tools.logging :as log]
[clojure.string :as s]
[environ.core :refer [env]]
[metabase.driver.generic-sql :as sql]
(metabase.test.data [generic-sql :as generic]
[interface :as i]
[postgres :as postgres])
[metabase.util :as u])
(:import metabase.driver.redshift.RedshiftDriver))
;; Time, UUID types aren't supported by redshift
(def ^:private ^:const field-base-type->sql-type
{:BigIntegerField "BIGINT"
:BooleanField "BOOL"
:CharField "VARCHAR(254)"
:DateField "DATE"
:DateTimeField "TIMESTAMP"
:DecimalField "DECIMAL"
:FloatField "FLOAT8"
:IntegerField "INTEGER"
:TextField "TEXT"})
(defn- get-db-env-var
"Look up the relevant env var for AWS connection details or throw an exception if it's not set.
(get-db-env-var :user) ; Look up `MB_REDSHIFT_USER`"
[env-var & [default]]
(or (env (keyword (format "mb-redshift-%s" (name env-var))))
default
(throw (Exception. (format "In order to test Redshift, you must specify the env var MB_REDSHIFT_%s."
(s/upper-case (name env-var)))))))
(def ^:private db-connection-details
(delay {:host (get-db-env-var :host)
:port (Integer/parseInt (get-db-env-var :port "5439"))
:db (get-db-env-var :db)
:user (get-db-env-var :user)
:password (get-db-env-var :password)}))
;; Redshift is tested remotely, which means we need to support multiple tests happening against the same remote host at the same time.
;; Since Redshift doesn't let us create and destroy databases (we must re-use the same database throughout the tests) we'll just fake it
;; by creating a new schema when tests start running and re-use the same schema for each test
(defonce ^:const session-schema-number
(rand-int 240)) ; there's a maximum of 256 schemas per DB so make sure we don't go over that limit
(defonce ^:const session-schema-name
(str "schema_" session-schema-number))
(defn- qualified-name-components
([_ db-name]
[db-name])
([_ _ table-name]
[session-schema-name table-name])
([_ _ table-name field-name]
[session-schema-name table-name field-name]))
(extend RedshiftDriver
generic/IGenericSQLDatasetLoader
(merge generic/DefaultsMixin
{:create-db-sql (constantly nil)
:drop-db-if-exists-sql (constantly nil)
:drop-table-if-exists-sql generic/drop-table-if-exists-cascade-sql
:field-base-type->sql-type (fn [_ base-type]
(field-base-type->sql-type base-type))
:pk-sql-type (constantly "INTEGER IDENTITY(1,1)")
:qualified-name-components qualified-name-components})
i/IDatasetLoader
(merge generic/IDatasetLoaderMixin
{:database->connection-details (fn [& _]
@db-connection-details)
:engine (constantly :redshift)}))
;;; Create + destroy the schema used for this test session
(defn- execute-when-testing-redshift! [format-str & args]
(when (contains? @(resolve 'metabase.test.data.datasets/test-engines) :redshift)
(let [sql (apply format format-str args)]
(log/info (u/format-color 'blue "[Redshift] %s" sql))
(jdbc/execute! (sql/connection-details->spec (RedshiftDriver.) @db-connection-details)
[sql]))))
(defn- create-session-schema!
{:expectations-options :before-run}
[]
(execute-when-testing-redshift! "DROP SCHEMA IF EXISTS %s CASCADE; CREATE SCHEMA %s;" session-schema-name session-schema-name))
(defn- destroy-session-schema!
{:expectations-options :after-run}
[]
(execute-when-testing-redshift! "DROP SCHEMA IF EXISTS %s CASCADE;" session-schema-name))
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment