Skip to content
Snippets Groups Projects
Commit 0556b234 authored by Cam Saul's avatar Cam Saul
Browse files

work towards testing w/ a Postgres test data set

parent d1006f32
No related branches found
No related tags found
No related merge requests found
......@@ -11,5 +11,5 @@ test:
# 2) runs Eastwood linter + Bikeshed linter
# 3) runs JS linter + JS test
# 4) runs lein uberjar
- case $CIRCLE_NODE_INDEX in 0) MB_TEST_DATASETS=generic-sql,mongo lein test ;; 1) MB_DB_TYPE=postgres MB_DB_DBNAME=circle_test MB_DB_PORT=5432 MB_DB_USER=ubuntu MB_DB_HOST=localhost lein test ;; 2) lein eastwood && lein bikeshed --max-line-length 240 ;; 3) npm run lint && npm run build && npm run test ;; 4) CI_DISABLE_WEBPACK_MINIFICATION=1 lein uberjar ;; esac:
- case $CIRCLE_NODE_INDEX in 0) MB_TEST_DATASETS=generic-sql,mongo,postgres lein test ;; 1) MB_DB_TYPE=postgres MB_DB_DBNAME=circle_test MB_DB_PORT=5432 MB_DB_USER=ubuntu MB_DB_HOST=localhost lein test ;; 2) lein eastwood && lein bikeshed --max-line-length 240 ;; 3) npm run lint && npm run build && npm run test ;; 4) CI_DISABLE_WEBPACK_MINIFICATION=1 lein uberjar ;; esac:
parallel: true
......@@ -38,25 +38,32 @@
(binding [*query* query]
(try
;; Process the expanded query and generate a korma form
(let [korma-form `(let [entity# (korma-entity ~database ~source-table)]
(select entity# ~@(->> (map apply-form (:query query))
(filter identity)
(mapcat #(if (vector? %) % [%])))))]
(let [entity (gensym "ENTITY_")
korma-select-form `(select ~entity ~@(->> (map apply-form (:query query))
(filter identity)
(mapcat #(if (vector? %) % [%]))))
set-timezone-sql (when-let [timezone (:timezone (:details database))]
(when-let [set-timezone-sql (:timezone->set-timezone-sql qp/*driver*)]
`(exec-raw ~(set-timezone-sql timezone))))
korma-form `(let [~entity (korma-entity ~database ~source-table)]
~(if set-timezone-sql `(korma.db/with-db (:db ~entity)
(korma.db/transaction
~set-timezone-sql
~korma-select-form))
korma-select-form))]
;; Log generated korma form
(when (config/config-bool :mb-db-logging)
(log-korma-form korma-form))
;; Now eval the korma form. Then annotate the results
;; TODO - why does this happen within the individual drivers still? Annotate should be moved out
(let [results (eval korma-form)]
{:results results
:uncastify-fn uncastify}))
(catch java.sql.SQLException e
(let [^String message (or (->> (.getMessage e) ; error message comes back like "Error message ... [status-code]" sometimes
(let [^String message (or (->> (.getMessage e) ; error message comes back like "Error message ... [status-code]" sometimes
(re-find #"(?s)(^.*)\s+\[[\d-]+\]$") ; status code isn't useful and makes unit tests hard to write so strip it off
second) ; (?s) = Pattern.DOTALL - tell regex `.` to match newline characters as well
second) ; (?s) = Pattern.DOTALL - tell regex `.` to match newline characters as well
(.getMessage e))]
(throw (Exception. message)))))))
......
......@@ -100,7 +100,6 @@
:connection-details->connection-spec connection-details->connection-spec
:database->connection-details database->connection-details
:sql-string-length-fn :LENGTH
:timezone->set-timezone-sql nil
:cast-timestamp-seconds-field-to-date-fn cast-timestamp-seconds-field-to-date-fn
:cast-timestamp-milliseconds-field-to-date-fn cast-timestamp-milliseconds-field-to-date-fn
:uncastify-timestamp-regex uncastify-timestamp-regex}))
......@@ -111,15 +111,16 @@
(defn- cast-timestamp-seconds-field-to-date-fn [table-name field-name]
{:pre [(string? table-name)
(string? field-name)]}
(format "CAST(TO_TIMESTAMP(\"%s\".\"%s\") AS DATE)" table-name field-name))
(format "(TIMESTAMP WITH TIME ZONE 'epoch' + (\"%s\".\"%s\" * INTERVAL '1 second'))::date" table-name field-name))
(defn- cast-timestamp-milliseconds-field-to-date-fn [table-name field-name]
{:pre [(string? table-name)
(string? field-name)]}
(format "CAST(TO_TIMESTAMP(\"%s\".\"%s\" / 1000) AS DATE)" table-name field-name))
(format "(TIMESTAMP WITH TIME ZONE 'epoch' + (\"%s\".\"%s\" * INTERVAL '1 millisecond'))::date" table-name field-name))
(def ^:private ^:const uncastify-timestamp-regex
#"CAST\(TO_TIMESTAMP\([^.\s]+\.([^.\s]+)(?: / 1000)?\) AS DATE\)")
;; TODO - this doesn't work
#"TO_TIMESTAMP\([^.\s]+\.([^.\s]+)(?: / 1000)?\)::date")
;; ## DRIVER
......
......@@ -342,9 +342,11 @@
(= col-kw :count) {:base_type :IntegerField
:special_type :number}
;; Otherwise something went wrong !
:else (throw (Exception. (format "Annotation failed: don't know what to do with Field '%s'.\nExpected these Fields:\n%s"
col-kw
(u/pprint-to-str field-kw->field))))))))
:else (do (log/error (u/format-color 'red "Annotation failed: don't know what to do with Field '%s'.\nExpected these Fields:\n%s"
col-kw
(u/pprint-to-str field-kw->field)))
{:base_type :UnknownField
:special_type nil})))))
;; Add FK info the the resulting Fields
add-fields-extra-info)))
......
......@@ -782,7 +782,7 @@
:query ~query})))
;; There were 9 "sad toucan incidents" on 2015-06-02
(datasets/expect-with-datasets #{:generic-sql}
(datasets/expect-with-datasets #{:generic-sql :postgres}
9
(->> (query-with-temp-db defs/sad-toucan-incidents
:source_table &incidents:id
......@@ -794,25 +794,42 @@
;;; Unix timestamp breakouts -- SQL only
(datasets/expect-with-datasets #{:generic-sql}
[["2015-06-01" 6]
["2015-06-02" 9]
["2015-06-03" 5]
["2015-06-04" 9]
["2015-06-05" 8]
["2015-06-06" 9]
["2015-06-07" 8]
["2015-06-08" 9]
["2015-06-09" 7]
["2015-06-10" 8]]
(->> (query-with-temp-db defs/sad-toucan-incidents
:source_table &incidents:id
:aggregation ["count"]
:breakout [&incidents.timestamp:id]
:limit 10)
:data :rows
(map (fn [[^java.util.Date date count]]
[(.toString date) (int count)]))))
(let [do-query (fn [] (->> (query-with-temp-db defs/sad-toucan-incidents
:source_table &incidents:id
:aggregation ["count"]
:breakout [&incidents.timestamp:id]
:limit 10)
:data :rows
(map (fn [[^java.util.Date date count]]
[(.toString date) (int count)]))))]
(datasets/expect-with-dataset :generic-sql
[["2015-06-01" 6]
["2015-06-02" 9]
["2015-06-03" 5]
["2015-06-04" 9]
["2015-06-05" 8]
["2015-06-06" 9]
["2015-06-07" 8]
["2015-06-08" 9]
["2015-06-09" 7]
["2015-06-10" 8]]
(do-query))
;; postgres gives us *slightly* different answers because I think it's actually handling UNIX timezones properly (with timezone = UTC)
;; as opposed to H2 which is giving us the wrong timezome. TODO - verify this
(datasets/expect-with-dataset :postgres
[["2015-06-01" 8]
["2015-06-02" 9]
["2015-06-03" 9]
["2015-06-04" 4]
["2015-06-05" 11]
["2015-06-06" 8]
["2015-06-07" 6]
["2015-06-08" 10]
["2015-06-09" 6]
["2015-06-10" 10]]
(do-query)))
;; +------------------------------------------------------------------------------------------------------------------------+
......
......@@ -11,7 +11,8 @@
[table :refer [Table]])
(metabase.test.data [data :as data]
[h2 :as h2]
[mongo :as mongo])
[mongo :as mongo]
[postgres :as postgres])
[metabase.util :as u]))
;; # IDataset
......@@ -88,10 +89,11 @@
(def ^:private generic-sql-db
(delay ))
(deftype GenericSqlDriverData [dbpromise]
(deftype GenericSqlDriverData [dataset-loader-fn
dbpromise]
IDataset
(dataset-loader [_]
(h2/dataset-loader))
(dataset-loader-fn))
(load-data! [this]
(when-not (realized? dbpromise)
......@@ -128,7 +130,10 @@
(def dataset-name->dataset
"Map of dataset keyword name -> dataset instance (i.e., an object that implements `IDataset`)."
{:mongo (MongoDriverData.)
:generic-sql (GenericSqlDriverData. (promise))})
:generic-sql (GenericSqlDriverData. h2/dataset-loader (promise))
;; TODO - make sure we have pg connection info
:postgres (GenericSqlDriverData. postgres/dataset-loader (promise))})
(def ^:const all-valid-dataset-names
"Set of names of all valid datasets."
......
(ns metabase.test.data.generic-sql
"Common functionality for various Generic SQL dataset loaders."
(:require [clojure.tools.logging :as log]
[korma.core :as k]
[metabase.test.data.interface :as i])
(:import (metabase.test.data.interface DatabaseDefinition
TableDefinition)))
(defprotocol IGenericSQLDatasetLoader
"Methods that generic SQL dataset loaders should implement so they can use the shared functions in `metabase.test.data.generic-sql`."
(execute-sql! [this ^DatabaseDefinition database-definition ^String raw-sql]
"Execute RAW-SQL against database defined by DATABASE-DEFINITION.")
(korma-entity [this ^DatabaseDefinition database-definition ^TableDefinition table-definition]
"Return a Korma entity (e.g., one that can be passed to `select` or `sel` for the table
defined by TABLE-DEFINITION in the database defined by DATABASE-DEFINITION.")
(pk-sql-type ^String [this]
"SQL that should be used for creating the PK Table ID, e.g. `SERIAL` or `BIGINT AUTOINCREMENT`.")
(field-base-type->sql-type ^String [this base-type]
"Given a `Field.base_type`, return the SQL type we should use for that column when creating a DB."))
(defn create-physical-table! [dataset-loader database-definition {:keys [table-name field-definitions], :as table-definition}]
;; Drop the table if it already exists
(i/drop-physical-table! dataset-loader database-definition table-definition)
;; Now create the new table
(execute-sql! dataset-loader database-definition
(format "CREATE TABLE \"%s\" (%s, \"ID\" %s, PRIMARY KEY (\"ID\"));"
table-name
(->> field-definitions
(map (fn [{:keys [field-name base-type]}]
(format "\"%s\" %s" field-name (field-base-type->sql-type dataset-loader base-type))))
(interpose ", ")
(apply str))
(pk-sql-type dataset-loader))))
(defn drop-physical-table! [dataset-loader database-definition table-definition]
(execute-sql! dataset-loader database-definition
(format "DROP TABLE IF EXISTS \"%s\";" (:table-name table-definition))))
(defn create-physical-db! [dataset-loader {:keys [table-definitions], :as database-definition}]
;; Create all the Tables
(doseq [^TableDefinition table-definition table-definitions]
(log/info (format "Creating table '%s'..." (:table-name table-definition)))
(i/create-physical-table! dataset-loader database-definition table-definition))
;; Now add the foreign key constraints
(doseq [{:keys [table-name field-definitions]} table-definitions]
(doseq [{dest-table-name :fk, field-name :field-name} field-definitions]
(when dest-table-name
(execute-sql! dataset-loader database-definition
(format "ALTER TABLE \"%s\" ADD CONSTRAINT \"FK_%s_%s\" FOREIGN KEY (\"%s\") REFERENCES \"%s\" (\"ID\");"
table-name
field-name dest-table-name
field-name
(name dest-table-name)))))))
(defn load-table-data! [dataset-loader database-definition table-definition]
(let [rows (:rows table-definition)
fields-for-insert (map :field-name (:field-definitions table-definition))]
(println (korma-entity dataset-loader database-definition table-definition))
(-> (korma-entity dataset-loader database-definition table-definition)
(k/insert (k/values (->> (for [row rows]
(for [v row]
(if (instance? java.util.Date v) (java.sql.Timestamp. (.getTime ^java.util.Date v))
v)))
(map (partial zipmap fields-for-insert))))))))
......@@ -5,69 +5,88 @@
[clojure.string :as s]
(korma [core :as k]
[db :as kdb])
[metabase.test.data.interface :refer :all])
(metabase.test.data [generic-sql :as generic]
[interface :refer :all]))
(:import (metabase.test.data.interface DatabaseDefinition
FieldDefinition
TableDefinition)))
(def ^:private ^:const field-base-type->sql-type
{:BigIntegerField "BIGINT"
:BooleanField "BOOL"
:CharField "VARCHAR(254)"
:DateField "DATE"
:DateTimeField "DATETIME"
:DecimalField "DECIMAL"
:FloatField "FLOAT"
:IntegerField "INTEGER"
:TextField "TEXT"
:TimeField "TIME"})
;; ## DatabaseDefinition helper functions
(defn filename
(defn- filename
"Return filename that should be used for connecting to H2 database defined by DATABASE-DEFINITION.
This does not include the `.mv.db` extension."
[^DatabaseDefinition database-definition]
(format "%s/target/%s" (System/getProperty "user.dir") (escaped-name database-definition)))
(defn connection-details
(defn- connection-details
"Return a Metabase `Database.details` for H2 database defined by DATABASE-DEFINITION."
[^DatabaseDefinition database-definition]
{:db (format (if (:short-lived? database-definition) "file:%s" ; for short-lived connections don't create a server thread and don't use a keep-alive connection
"file:%s;AUTO_SERVER=TRUE;DB_CLOSE_DELAY=-1")
(filename database-definition))})
(defn korma-connection-pool
(defn- korma-connection-pool
"Return an H2 korma connection pool to H2 database defined by DATABASE-DEFINITION."
[^DatabaseDefinition database-definition]
(kdb/create-db (kdb/h2 (assoc (connection-details database-definition)
:naming {:keys s/lower-case
:fields s/upper-case}))))
:naming {:keys s/lower-case
:fields s/upper-case}))))
(defn exec-sql
"Execute RAW-SQL against H2 instance of H2 database defined by DATABASE-DEFINITION."
[^DatabaseDefinition database-definition ^String raw-sql]
(log/info raw-sql)
(k/exec-raw (korma-connection-pool database-definition) raw-sql))
;; ## Implementation
(defprotocol IH2DatasetFormat
(format-for-h2 [this]
"Format dataset definitions for H2, e.g. upcasing `Table` and `Field` names."))
;; ## TableDefinition helper functions
(extend-protocol IH2DatasetFormat
DatabaseDefinition
(format-for-h2 [this]
(update-in this [:table-definitions] (partial map format-for-h2)))
(defn korma-entity
"Return a Korma entity (e.g., one that can be passed to `select` or `sel` for the table
defined by TABLE-DEFINITION in the H2 database defined by DATABASE-DEFINITION."
[^TableDefinition table-definition ^DatabaseDefinition database-definition]
(-> (k/create-entity (:table-name table-definition))
(k/database (korma-connection-pool database-definition))))
TableDefinition
(format-for-h2 [this]
(-> this
(update-in [:table-name] s/upper-case)
(update-in [:field-definitions] (partial map format-for-h2))))
FieldDefinition
(format-for-h2 [this]
(cond-> (update-in this [:field-name] s/upper-case)
(:pk this) (update-in [:pk] (comp s/upper-case name)))))
;; ## Internal Stuff
(def ^:private ^:const field-base-type->sql-type
"Map of `Field.base_type` to the SQL type we should use for that column when creating a DB."
{:BigIntegerField "BIGINT"
:BooleanField "BOOL"
:CharField "VARCHAR(254)"
:DateField "DATE"
:DateTimeField "DATETIME"
:DecimalField "DECIMAL"
:FloatField "FLOAT"
:IntegerField "INTEGER"
:TextField "TEXT"
:TimeField "TIME"})
;; ## Public Concrete DatasetLoader instance
;; For some reason this doesn't seem to work if we define IDatasetLoader methods inline, but does work when we explicitly use extend-protocol
(defrecord H2DatasetLoader [])
(defrecord H2DatasetLoader []
generic/IGenericSQLDatasetLoader
(generic/execute-sql! [_ database-definition raw-sql]
(log/info raw-sql)
(k/exec-raw (korma-connection-pool database-definition) raw-sql))
(generic/korma-entity [_ database-definition table-definition]
(-> (k/create-entity (:table-name table-definition))
(k/database (korma-connection-pool database-definition))))
(generic/pk-sql-type [_]
"BIGINT AUTO_INCREMENT")
(generic/field-base-type->sql-type [_ field-type]
(field-base-type->sql-type field-type)))
(extend-protocol IDatasetLoader
H2DatasetLoader
(engine [_]
......@@ -82,54 +101,16 @@
(.delete file))))
(create-physical-table! [this database-definition table-definition]
;; Drop the table if it already exists
(drop-physical-table! this database-definition table-definition)
;; Now create the new table
(exec-sql
database-definition
(format "CREATE TABLE \"%s\" (%s, \"ID\" BIGINT AUTO_INCREMENT, PRIMARY KEY (\"ID\"));"
(s/upper-case (:table-name table-definition))
(->> (:field-definitions table-definition)
(map (fn [{:keys [field-name base-type]}]
(format "\"%s\" %s" (s/upper-case field-name) (base-type field-base-type->sql-type))))
(interpose ", ")
(apply str)))))
(generic/create-physical-table! this database-definition (format-for-h2 table-definition)))
(create-physical-db! [this database-definition]
;; Create all the Tables
(doseq [^TableDefinition table-definition (:table-definitions database-definition)]
(log/info (format "Creating table '%s'..." (:table-name table-definition)))
(create-physical-table! this database-definition table-definition))
;; Now add the foreign key constraints
(doseq [^TableDefinition table-definition (:table-definitions database-definition)]
(let [table-name (s/upper-case (:table-name table-definition))]
(doseq [{dest-table-name :fk, field-name :field-name} (:field-definitions table-definition)]
(when dest-table-name
(let [field-name (s/upper-case field-name)
dest-table-name (s/upper-case (name dest-table-name))]
(exec-sql
database-definition
(format "ALTER TABLE \"%s\" ADD CONSTRAINT IF NOT EXISTS \"FK_%s_%s\" FOREIGN KEY (\"%s\") REFERENCES \"%s\" (\"ID\");"
table-name
field-name dest-table-name
field-name
dest-table-name))))))))
(load-table-data! [_ database-definition table-definition]
(let [rows (:rows table-definition)
fields-for-insert (map :field-name (:field-definitions table-definition))]
(-> (korma-entity table-definition database-definition)
(k/insert (k/values (map (partial zipmap fields-for-insert)
rows))))))
(drop-physical-table! [_ database-definition table-definition]
(exec-sql
database-definition
(format "DROP TABLE IF EXISTS \"%s\";" (s/upper-case (:table-name table-definition))))))
(generic/create-physical-db! this (format-for-h2 database-definition)))
(load-table-data! [this database-definition table-definition]
(generic/load-table-data! this database-definition table-definition))
(drop-physical-table! [this database-definition table-definition]
(generic/drop-physical-table! this database-definition (format-for-h2 table-definition))))
(defn dataset-loader []
(let [loader (->H2DatasetLoader)]
(assert (satisfies? IDatasetLoader loader))
loader))
(->H2DatasetLoader))
(ns metabase.test.data.postgres
"Code for creating / destroying a Postgres database from a `DatabaseDefinition`."
(:require [clojure.java.jdbc :as jdbc]
[clojure.tools.logging :as log]
[environ.core :refer [env]]
(korma [core :as k]
[db :as kdb])
(metabase.test.data [generic-sql :as generic]
[interface :refer :all]))
(:import (metabase.test.data.interface DatabaseDefinition
FieldDefinition
TableDefinition)))
(def ^:private ^:const field-base-type->sql-type
{:BigIntegerField "BIGINT"
:BooleanField "BOOL"
:CharField "VARCHAR(254)"
:DateField "DATE"
:DateTimeField "TIMESTAMP"
:DecimalField "DECIMAL"
:FloatField "FLOAT"
:IntegerField "INTEGER"
:TextField "TEXT"
:TimeField "TIME"})
(defn- pg-connection-details [^DatabaseDefinition database-definition]
(merge {:host "localhost"
:port 5432}
;; HACK
(when (env :circleci)
{:user "ubuntu"})))
(defn- db-connection-details [^DatabaseDefinition database-definition]
(assoc (pg-connection-details database-definition)
:db (:database-name database-definition)))
(defn- execute! [scope ^DatabaseDefinition database-definition & format-strings]
(jdbc/execute! (-> ((case scope
:pg pg-connection-details
:db db-connection-details) database-definition)
kdb/postgres
(assoc :make-pool? false))
[(apply format format-strings)]
:transaction? false))
(defrecord PostgresDatasetLoader []
generic/IGenericSQLDatasetLoader
(generic/execute-sql! [_ database-definition raw-sql]
(log/info raw-sql)
(execute! :db database-definition raw-sql))
(generic/korma-entity [_ database-definition table-definition]
(-> (k/create-entity (:table-name table-definition))
(k/database (-> (db-connection-details database-definition)
kdb/postgres
(assoc :make-pool? false)
kdb/create-db))))
(generic/pk-sql-type [_]
"SERIAL")
(generic/field-base-type->sql-type [_ field-type]
(field-base-type->sql-type field-type)))
(extend-protocol IDatasetLoader
PostgresDatasetLoader
(engine [_]
:postgres)
(database->connection-details [_ database-definition]
(assoc (db-connection-details database-definition)
:timezone :America/Los_Angeles))
(drop-physical-db! [_ database-definition]
(execute! :pg database-definition "DROP DATABASE IF EXISTS \"%s\";" (:database-name database-definition)))
(drop-physical-table! [this database-definition table-definition]
(generic/drop-physical-table! this database-definition table-definition))
(create-physical-table! [this database-definition table-definition]
(generic/create-physical-table! this database-definition table-definition))
(create-physical-db! [this {:keys [database-name], :as database-definition}]
(execute! :pg database-definition "DROP DATABASE IF EXISTS \"%s\";" database-name)
(execute! :pg database-definition "CREATE DATABASE \"%s\";" database-name)
;; double check that we can connect to the newly created DB
(metabase.driver/can-connect-with-details? :postgres (db-connection-details database-definition) :rethrow-exceptions)
;; call the generic implementation to create Tables + FKs
(generic/create-physical-db! this database-definition))
(load-table-data! [this database-definition table-definition]
(generic/load-table-data! this database-definition table-definition)))
(defn dataset-loader []
(->PostgresDatasetLoader))
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment