From c7831c5c1b28312c2156da70697ef6bf5f29e3b2 Mon Sep 17 00:00:00 2001
From: Cam Saul <cam@geotip.com>
Date: Wed, 29 Apr 2015 20:04:41 -0700
Subject: [PATCH] rework some of the test data stuff
---
.dir-locals.el | 1 -
src/metabase/db.clj | 14 +--
.../driver/generic_sql/query_processor.clj | 7 +-
src/metabase/driver/mongo/query_processor.clj | 9 +-
src/metabase/driver/query_processor.clj | 3 +
src/metabase/driver/sync.clj | 87 ++++++-------
src/metabase/models/database.clj | 4 +-
test/metabase/api/meta/table_test.clj | 21 +++-
test/metabase/api/query_test.clj | 2 +-
test/metabase/api/result_test.clj | 2 +-
test/metabase/driver/mongo/test_data.clj | 56 ++++++---
test/metabase/driver/mongo_test.clj | 46 ++++---
test/metabase/driver/query_processor_test.clj | 23 ++--
test/metabase/test/data.clj | 116 ++++++++++++++++++
test/metabase/test/data/h2.clj | 11 ++
test/metabase/test/data/interface.clj | 17 +++
test/metabase/test/data/mongo.clj | 18 +++
test/metabase/test_setup.clj | 15 ++-
18 files changed, 328 insertions(+), 124 deletions(-)
create mode 100644 test/metabase/test/data.clj
create mode 100644 test/metabase/test/data/h2.clj
create mode 100644 test/metabase/test/data/interface.clj
create mode 100644 test/metabase/test/data/mongo.clj
diff --git a/.dir-locals.el b/.dir-locals.el
index ee8ad0ec4f3..ef2abb2e2b4 100644
--- a/.dir-locals.el
+++ b/.dir-locals.el
@@ -20,7 +20,6 @@
(expect-expansion 0)
(expect-let 1)
(expect-with-all-drivers 1)
- (expect-with-data-loaded 1)
(ins 1)
(let-400 1)
(let-404 1)
diff --git a/src/metabase/db.clj b/src/metabase/db.clj
index a07581e3314..0d801462892 100644
--- a/src/metabase/db.clj
+++ b/src/metabase/db.clj
@@ -74,12 +74,6 @@
(config/config-str :mb-db-user)
(config/config-str :mb-db-pass))))
-(defn test-db-conn
- "Simple test of a JDBC connection."
- [jdbc-db]
- (let [result (first (jdbc/query jdbc-db ["select 7 as num"] :row-fn :num))]
- (assert (= 7 result) "JDBC Connection Test FAILED")))
-
;; ## MIGRATE
@@ -98,16 +92,22 @@
(def ^:private setup-db-has-been-called?
(atom false))
+(def ^:private db-can-connect? (u/runtime-resolved-fn 'metabase.driver 'can-connect?))
+
(defn setup-db
"Do general perparation of database by validating that we can connect.
Caller can specify if we should run any pending database migrations."
[& {:keys [auto-migrate]
:or {auto-migrate true}}]
(reset! setup-db-has-been-called? true)
+ (log/info "Setting up DB specs...")
(let [jdbc-db (setup-jdbc-db)
korma-db (setup-korma-db)]
;; Test DB connection and throw exception if we have any troubles connecting
- (test-db-conn jdbc-db)
+ (log/info "Verifying Database Connection ...")
+ (assert (db-can-connect? {:engine (config/config-kw :mb-db-type)
+ :details {:conn_str (metabase-db-conn-str)}})
+ "Unable to connect to Metabase DB.")
(log/info "Verify Database Connection ... CHECK")
;; Run through our DB migration process and make sure DB is fully prepared
(if auto-migrate
diff --git a/src/metabase/driver/generic_sql/query_processor.clj b/src/metabase/driver/generic_sql/query_processor.clj
index af7f0c73cce..2398f7d09ff 100644
--- a/src/metabase/driver/generic_sql/query_processor.clj
+++ b/src/metabase/driver/generic_sql/query_processor.clj
@@ -5,9 +5,10 @@
[korma.core :refer :all]
[metabase.config :as config]
[metabase.db :refer :all]
- [metabase.driver.generic-sql.native :as native]
+ [metabase.driver.query-processor :as qp]
+ (metabase.driver.generic-sql [native :as native]
+ [util :refer :all])
[metabase.driver.generic-sql.query-processor.annotate :as annotate]
- [metabase.driver.generic-sql.util :refer :all]
(metabase.models [database :refer [Database]]
[field :refer [Field]]
[table :refer [Table]])))
@@ -234,7 +235,7 @@
(defn- log-query
"Log QUERY Dictionary and the korma form and SQL that the Query Processor translates it to."
[{:keys [source_table] :as query} forms]
- (when-not *jdbc-metadata* ; HACK. If *jdbc-metadata* is bound we're probably doing a DB sync. Don't log its hundreds of QP calls, which make it hard to debug.
+ (when-not qp/*disable-qp-logging*
(log/debug
"\n********************"
"\nSOURCE TABLE: " source_table
diff --git a/src/metabase/driver/mongo/query_processor.clj b/src/metabase/driver/mongo/query_processor.clj
index 94d4903e4c4..27738f5ca7f 100644
--- a/src/metabase/driver/mongo/query_processor.clj
+++ b/src/metabase/driver/mongo/query_processor.clj
@@ -1,6 +1,7 @@
(ns metabase.driver.mongo.query-processor
(:refer-clojure :exclude [find sort])
(:require [clojure.core.match :refer [match]]
+ [clojure.tools.logging :as log]
[colorize.core :as color]
(monger [collection :as mc]
[core :as mg]
@@ -32,10 +33,10 @@
(with-mongo-connection [_ (sel :one :fields [Database :details] :id database-id)]
(case (keyword query-type)
:query (let [generated-query (process-structured (:query query))]
- ;; ; TODO - log/debug
- (println (color/magenta "\n******************** Generated Monger Query: ********************\n"
- (with-out-str (clojure.pprint/pprint generated-query))
- "*****************************************************************\n"))
+ (when-not qp/*disable-qp-logging*
+ (log/debug (color/magenta "\n******************** Generated Monger Query: ********************\n"
+ (with-out-str (clojure.pprint/pprint generated-query))
+ "*****************************************************************\n")))
(->> (eval generated-query)
(annotate-results (:query query))))
:native (->> (eval-raw-command (:query (:native query)))
diff --git a/src/metabase/driver/query_processor.clj b/src/metabase/driver/query_processor.clj
index 6ecdb14c17d..c17a1fd415a 100644
--- a/src/metabase/driver/query_processor.clj
+++ b/src/metabase/driver/query_processor.clj
@@ -11,6 +11,9 @@
(def ^:dynamic *query* "The structured query we're currently processing, before any preprocessing occurs (i.e. the `:query` part of the API call body)"
nil)
+(def ^:dynamic *disable-qp-logging* "Should we disable logging for the QP? (e.g., during sync we probably want to turn it off to keep logs less cluttered)."
+ false)
+
(defn preprocess [{query-type :type :as query}]
(case (keyword query-type)
:query (preprocess-structured query)
diff --git a/src/metabase/driver/sync.clj b/src/metabase/driver/sync.clj
index 59e42ace339..d2b7ee6b45f 100644
--- a/src/metabase/driver/sync.clj
+++ b/src/metabase/driver/sync.clj
@@ -5,7 +5,8 @@
[colorize.core :as color]
[korma.core :as k]
[metabase.db :refer :all]
- [metabase.driver.interface :refer :all]
+ (metabase.driver [interface :refer :all]
+ [query-processor :as qp])
[metabase.driver.sync.queries :as queries]
(metabase.models [field :refer [Field] :as field]
[foreign-key :refer [ForeignKey]]
@@ -27,52 +28,54 @@
(defn sync-database!
"Sync DATABASE and all its Tables and Fields."
[driver database]
- (sync-in-context driver database
- (fn []
- (log/info (color/blue (format "Syncing database %s..." (:name database))))
-
- (let [active-table-names (active-table-names driver database)
- table-name->id (sel :many :field->id [Table :name] :db_id (:id database) :active true)]
- (assert (set? active-table-names) "active-table-names should return a set.")
- (assert (every? string? active-table-names) "active-table-names should return the names of Tables as *strings*.")
-
- ;; First, let's mark any Tables that are no longer active as such.
- ;; These are ones that exist in table-name->id but not in active-table-names.
- (log/debug "Marking inactive tables...")
- (doseq [[table-name table-id] table-name->id]
- (when-not (contains? active-table-names table-name)
- (upd Table table-id :active false)
- (log/info (format "Marked table %s.%s as inactive." (:name database) table-name))
-
- ;; We need to mark driver Table's Fields as inactive so we don't expose them in UI such as FK selector (etc.) This can happen in the background
- (future (k/update Field
- (k/where {:table_id table-id})
- (k/set-fields {:active false})))))
-
- ;; Next, we'll create new Tables (ones that came back in active-table-names but *not* in table-name->id)
- (log/debug "Creating new tables...")
- (let [existing-table-names (set (keys table-name->id))]
- (doseq [active-table-name active-table-names]
- (when-not (contains? existing-table-names active-table-name)
- (ins Table :db_id (:id database), :active true, :name active-table-name)
- (log/info (format "Found new table: %s.%s" (:name database) active-table-name))))))
-
- ;; Now sync the active tables
- (log/debug "Syncing active tables...")
- (->> (sel :many Table :db_id (:id database) :active true)
- (map #(assoc % :db (delay database))) ; replace default delays with ones that reuse database (and don't require a DB call)
- (sync-database-active-tables! driver))
-
- (log/info (color/blue (format "Finished syncing database %s." (:name database)))))))
+ (binding [qp/*disable-qp-logging* true]
+ (sync-in-context driver database
+ (fn []
+ (log/info (color/blue (format "Syncing database %s..." (:name database))))
+
+ (let [active-table-names (active-table-names driver database)
+ table-name->id (sel :many :field->id [Table :name] :db_id (:id database) :active true)]
+ (assert (set? active-table-names) "active-table-names should return a set.")
+ (assert (every? string? active-table-names) "active-table-names should return the names of Tables as *strings*.")
+
+ ;; First, let's mark any Tables that are no longer active as such.
+ ;; These are ones that exist in table-name->id but not in active-table-names.
+ (log/debug "Marking inactive tables...")
+ (doseq [[table-name table-id] table-name->id]
+ (when-not (contains? active-table-names table-name)
+ (upd Table table-id :active false)
+ (log/info (format "Marked table %s.%s as inactive." (:name database) table-name))
+
+ ;; We need to mark driver Table's Fields as inactive so we don't expose them in UI such as FK selector (etc.) This can happen in the background
+ (future (k/update Field
+ (k/where {:table_id table-id})
+ (k/set-fields {:active false})))))
+
+ ;; Next, we'll create new Tables (ones that came back in active-table-names but *not* in table-name->id)
+ (log/debug "Creating new tables...")
+ (let [existing-table-names (set (keys table-name->id))]
+ (doseq [active-table-name active-table-names]
+ (when-not (contains? existing-table-names active-table-name)
+ (ins Table :db_id (:id database), :active true, :name active-table-name)
+ (log/info (format "Found new table: %s.%s" (:name database) active-table-name))))))
+
+ ;; Now sync the active tables
+ (log/debug "Syncing active tables...")
+ (->> (sel :many Table :db_id (:id database) :active true)
+ (map #(assoc % :db (delay database))) ; replace default delays with ones that reuse database (and don't require a DB call)
+ (sync-database-active-tables! driver))
+
+ (log/info (color/blue (format "Finished syncing database %s." (:name database))))))))
(defn sync-table!
"Sync a *single* TABLE by running all the sync steps for it.
This is used *instead* of `sync-database!` when syncing just one Table is desirable."
[driver table]
(let [database @(:db table)]
- (sync-in-context driver database
- (fn []
- (sync-database-active-tables! driver [table])))))
+ (binding [qp/*disable-qp-logging* true]
+ (sync-in-context driver database
+ (fn []
+ (sync-database-active-tables! driver [table]))))))
;; ### sync-database-active-tables! -- runs the sync-table steps over sequence of Tables
@@ -264,7 +267,6 @@
(extend-protocol ISyncDriverFieldPercentUrls ; Default implementation
Object
(field-percent-urls [this field]
- (log/warn (color/red (format "Using default (read: slow) implementation of field-percent-urls for driver %s." (.getName (class this)))))
(assert (extends? ISyncDriverFieldValues (class this))
"A sync driver implementation that doesn't implement ISyncDriverFieldPercentURLs must implement ISyncDriverFieldValues.")
(let [field-values (field-values-lazy-seq this field)]
@@ -312,7 +314,6 @@
(extend-protocol ISyncDriverFieldAvgLength ; Default implementation
Object
(field-avg-length [this field]
- (log/warn (color/red (format "Using default (read: slow) implementation of field-avg-length for driver %s." (.getName (class this)))))
(assert (extends? ISyncDriverFieldValues (class this))
"A sync driver implementation that doesn't implement ISyncDriverFieldAvgLength must implement ISyncDriverFieldValues.")
(let [field-values (field-values-lazy-seq this field)
diff --git a/src/metabase/models/database.clj b/src/metabase/models/database.clj
index e599e9174a0..195fb307592 100644
--- a/src/metabase/models/database.clj
+++ b/src/metabase/models/database.clj
@@ -18,7 +18,9 @@
:can_read (delay (org-can-read organization_id))
:can_write (delay (org-can-write organization_id))))
-(defmethod pre-cascade-delete Database [_ {:keys [id]}]
+(defmethod pre-cascade-delete Database [_ {:keys [id] :as database}]
+ (if (= (:engine database) :mongo)
+ (throw (Exception. "WHY ARE WE DESTROYING MONGO DB!")))
(cascade-delete 'metabase.models.table/Table :db_id id)
(cascade-delete 'metabase.models.query/Query :database_id id))
diff --git a/test/metabase/api/meta/table_test.clj b/test/metabase/api/meta/table_test.clj
index 5e94d113cac..2edb73465fd 100644
--- a/test/metabase/api/meta/table_test.clj
+++ b/test/metabase/api/meta/table_test.clj
@@ -2,6 +2,7 @@
"Tests for /api/meta/table endpoints."
(:require [expectations :refer :all]
[metabase.db :refer :all]
+ [metabase.driver.mongo.test-data :as mongo-data :refer [mongo-test-db-id]]
[metabase.http-client :as http]
[metabase.middleware.auth :as auth]
(metabase.models [field :refer [Field]]
@@ -21,10 +22,22 @@
;; ## GET /api/meta/table?org
;; These should come back in alphabetical order and include relevant metadata
-(expect [{:description nil, :entity_type nil, :name "CATEGORIES", :rows 75, :entity_name nil, :active true, :id (table->id :categories), :db_id @db-id}
- {:description nil, :entity_type nil, :name "CHECKINS", :rows 1000, :entity_name nil, :active true, :id (table->id :checkins), :db_id @db-id}
- {:description nil, :entity_type nil, :name "USERS", :rows 15, :entity_name nil, :active true, :id (table->id :users), :db_id @db-id}
- {:description nil, :entity_type nil, :name "VENUES", :rows 100, :entity_name nil, :active true, :id (table->id :venues), :db_id @db-id}]
+(expect [{:description nil, :entity_type nil, :name "CATEGORIES", :rows 75, :entity_name nil, :active true,
+ :id (table->id :categories), :db_id @db-id}
+ {:description nil, :entity_type nil, :name "CHECKINS", :rows 1000, :entity_name nil, :active true,
+ :id (table->id :checkins), :db_id @db-id}
+ {:description nil, :entity_type nil, :name "USERS", :rows 15, :entity_name nil, :active true,
+ :id (table->id :users), :db_id @db-id}
+ {:description nil, :entity_type nil, :name "VENUES", :rows 100, :entity_name nil, :active true,
+ :id (table->id :venues), :db_id @db-id}
+ {:description nil, :entity_type nil, :name "categories", :rows 75, :entity_name nil, :active true,
+ :id (mongo-data/table-name->id :categories), :db_id @mongo-test-db-id}
+ {:description nil, :entity_type nil, :name "checkins", :rows 1000, :entity_name nil, :active true,
+ :id (mongo-data/table-name->id :checkins), :db_id @mongo-test-db-id}
+ {:description nil, :entity_type nil, :name "users", :rows 15, :entity_name nil, :active true,
+ :id (mongo-data/table-name->id :users), :db_id @mongo-test-db-id}
+ {:description nil, :entity_type nil, :name "venues", :rows 100, :entity_name nil, :active true,
+ :id (mongo-data/table-name->id :venues), :db_id @mongo-test-db-id}]
(->> ((user->client :rasta) :get 200 "meta/table" :org @org-id)
(map #(dissoc % :db :created_at :updated_at)))) ; don't care about checking nested DB, and not sure how to compare `:created_at` / `:updated_at`
diff --git a/test/metabase/api/query_test.clj b/test/metabase/api/query_test.clj
index 12f01577040..46fb5026897 100644
--- a/test/metabase/api/query_test.clj
+++ b/test/metabase/api/query_test.clj
@@ -3,6 +3,7 @@
(:require [expectations :refer :all]
[korma.core :refer :all]
[metabase.db :refer :all]
+ [metabase.driver.mongo.test-data :as mongo-test-data]
(metabase.models [database :refer [Database]]
[query :refer [Query]]
[query-execution :refer [QueryExecution]])
@@ -35,7 +36,6 @@
{:name "Read Only", :id 1}
{:name "Read & Write", :id 2}]}
(do
- @test-db ; force lazy creation of test data / Metabase DB if it doesn't already exist
(cascade-delete Database :organization_id @org-id :id [not= (:id @test-db)]) ; delete any other rando test DBs made by other tests
((user->client :rasta) :get 200 "query/form_input" :org @org-id)))
diff --git a/test/metabase/api/result_test.clj b/test/metabase/api/result_test.clj
index 95de894379a..17b079cc87c 100644
--- a/test/metabase/api/result_test.clj
+++ b/test/metabase/api/result_test.clj
@@ -13,7 +13,7 @@
[]
(let [{query-id :id} (create-query)
query-execution ((user->client :rasta) :post 200 (format "query/%d" query-id))]
- (Thread/sleep 100) ; Give it 100ms to finish
+ (Thread/sleep 200) ; Give it 200ms to finish
query-execution))
;; ## GET /result/:id
diff --git a/test/metabase/driver/mongo/test_data.clj b/test/metabase/driver/mongo/test_data.clj
index eda1d5b6af7..00a89630217 100644
--- a/test/metabase/driver/mongo/test_data.clj
+++ b/test/metabase/driver/mongo/test_data.clj
@@ -29,24 +29,37 @@
;; ## MONGO-TEST-DB + OTHER DELAYS
-(def mongo-test-db
- "A delay that fetches or creates the Mongo test `Database`.
- If DB is created, `load-data` and `sync-database!` are called to get the DB in a state that we can use for testing."
- (delay (or (sel :one Database :name mongo-test-db-name)
- (let [db (ins Database
- :organization_id @org-id
- :name mongo-test-db-name
- :engine :mongo
- :details {:conn_str mongo-test-db-conn-str})]
- (log/debug (color/cyan "Loading Mongo test data..."))
- (load-data)
- (driver/sync-database! db)
- (log/debug (color/cyan "Done."))
- db))))
-
-(def mongo-test-db-id
- "A Delay that returns the ID of `mongo-test-db`, forcing creation of it if needed."
- (delay (:id @mongo-test-db)))
+(defn destroy!
+ "Remove `Database`, `Tables`, and `Fields` for the Mongo test DB."
+ []
+ #_(cascade-delete Database :name mongo-test-db-name))
+
+(defonce
+ ^{:doc "A delay that fetches or creates the Mongo test `Database`.
+ If DB is created, `load-data` and `sync-database!` are called to get the DB in a state that we can use for testing."}
+ mongo-test-db
+ (delay (let [db (or (sel :one Database :name mongo-test-db-name)
+ (let [db (ins Database
+ :organization_id @org-id
+ :name mongo-test-db-name
+ :engine :mongo
+ :details {:conn_str mongo-test-db-conn-str})]
+ (log/debug (color/cyan "Loading Mongo test data..."))
+ (load-data)
+ (driver/sync-database! db)
+ (log/debug (color/cyan "Done."))
+ db))]
+ (assert (and (map? db)
+ (integer? (:id db))
+ (exists? Database :id (:id db))))
+ db)))
+
+(defonce
+ ^{:doc "A Delay that returns the ID of `mongo-test-db`, forcing creation of it if needed."}
+ mongo-test-db-id
+ (delay (let [id (:id @mongo-test-db)]
+ (assert (integer? id))
+ id)))
;; ## FNS FOR GETTING RELEVANT TABLES / FIELDS
@@ -58,7 +71,12 @@
[table-name]
{:pre [(keyword? table-name)]
:post [(map? %)]}
- (sel :one Table :db_id @mongo-test-db-id :name (name table-name)))
+ (assert (exists? Database :id @mongo-test-db-id)
+ (format "Database with ID %d no longer exists!?" @mongo-test-db-id))
+ (or (sel :one Table :db_id @mongo-test-db-id :name (name table-name))
+ (println (colorize.core/red "db_id: " @mongo-test-db-id "\n"
+ "db: " (with-out-str (clojure.pprint/pprint (sel :one Database :id @mongo-test-db-id))) "\n"
+ "name: " (name table-name)))))
(def ^{:arglists '([table-name])} table-name->id
"Return ID of `Table` for Mongo test database (memoized).
diff --git a/test/metabase/driver/mongo_test.clj b/test/metabase/driver/mongo_test.clj
index 387c6e35828..bb94e19e935 100644
--- a/test/metabase/driver/mongo_test.clj
+++ b/test/metabase/driver/mongo_test.clj
@@ -39,13 +39,6 @@
[:venues :name]
[:venues :price]])
-(defmacro expect-with-data-loaded
- "Like `expect`, but forces the test database to be loaded/synced/etc. before running the test."
- [expected actual]
- `(expect (do @mongo-test-db
- ~expected)
- ~actual))
-
(defn- table-name->fake-table
"Return an object that can be passed like a `Table` to driver sync functions."
[table-name]
@@ -107,7 +100,7 @@
(i/active-table-names mongo/driver @mongo-test-db))
;; ### table->column-names
-(expect-with-data-loaded
+(expect
[#{:_id :name}
#{:_id :date :venue_id :user_id}
#{:_id :name :last_login}
@@ -117,7 +110,7 @@
(map table->column-names)))
;; ### field->base-type
-(expect-with-data-loaded
+(expect
[:IntegerField ; categories._id
:TextField ; categories.name
:IntegerField ; checkins._id
@@ -158,7 +151,7 @@
;; ## Big-picture tests for the way data should look post-sync
;; Test that Tables got synced correctly, and row counts are correct
-(expect-with-data-loaded
+(expect
[{:rows 75, :active true, :name "categories"}
{:rows 1000, :active true, :name "checkins"}
{:rows 15, :active true, :name "users"}
@@ -166,19 +159,22 @@
(sel :many :fields [Table :name :active :rows] :db_id @mongo-test-db-id (k/order :name)))
;; Test that Fields got synced correctly, and types are correct
-(expect-with-data-loaded
- [({:special_type :id, :base_type :IntegerField, :field_type :info, :active true, :name "_id"}
- {:special_type :category, :base_type :DateField, :field_type :info, :active true, :name "last_login"}
- {:special_type :category, :base_type :TextField, :field_type :info, :active true, :name "name"})
- ({:special_type :id, :base_type :IntegerField, :field_type :info, :active true, :name "_id"}
- {:special_type :category, :base_type :DateField, :field_type :info, :active true, :name "last_login"}
- {:special_type :category, :base_type :TextField, :field_type :info, :active true, :name "name"})
- ({:special_type :id, :base_type :IntegerField, :field_type :info, :active true, :name "_id"}
- {:special_type :category, :base_type :DateField, :field_type :info, :active true, :name "last_login"}
- {:special_type :category, :base_type :TextField, :field_type :info, :active true, :name "name"})
- ({:special_type :id, :base_type :IntegerField, :field_type :info, :active true, :name "_id"}
- {:special_type :category, :base_type :DateField, :field_type :info, :active true, :name "last_login"}
- {:special_type :category, :base_type :TextField, :field_type :info, :active true, :name "name"})]
+(expect
+ [[{:special_type :id, :base_type :IntegerField, :name "_id"}
+ {:special_type :category, :base_type :DateField, :name "last_login"}
+ {:special_type :category, :base_type :TextField, :name "name"}]
+ [{:special_type :id, :base_type :IntegerField, :name "_id"}
+ {:special_type :category, :base_type :DateField, :name "last_login"}
+ {:special_type :category, :base_type :TextField, :name "name"}]
+ [{:special_type :id, :base_type :IntegerField, :name "_id"}
+ {:special_type :category, :base_type :DateField, :name "last_login"}
+ {:special_type :category, :base_type :TextField, :name "name"}]
+ [{:special_type :id, :base_type :IntegerField, :name "_id"}
+ {:special_type :category, :base_type :DateField, :name "last_login"}
+ {:special_type :category, :base_type :TextField, :name "name"}]]
(let [table->fields (fn [table-name]
- (sel :many :fields [Field :name :active :field_type :base_type :special_type] :table_id (table-name->id :users) (k/order :name)))]
- (mapv table->fields table-names)))
+ (sel :many :fields [Field :name :base_type :special_type]
+ :active true
+ :table_id (table-name->id :users)
+ (k/order :name)))]
+ (map table->fields table-names)))
diff --git a/test/metabase/driver/query_processor_test.clj b/test/metabase/driver/query_processor_test.clj
index 1f1e4bd40c7..03a150a4f56 100644
--- a/test/metabase/driver/query_processor_test.clj
+++ b/test/metabase/driver/query_processor_test.clj
@@ -53,7 +53,8 @@
(binding [*driver-data* driver-data#
*db* db#
*db-id* (:id db#)]
- (assert (integer? *db-id*))
+ (assert (and (integer? *db-id*)
+ (map? *db*)))
~@body)))
(defmacro expect-with-all-drivers
@@ -82,16 +83,16 @@
:post [(integer? %)]}
(:id (->table table-name)))
-#_(def ^{:arglists '([table-name])} table-name->id
- "Given keyword TABLE-NAME, fetch ID of corresponding `Table` in `*db*`."
- (let [-table-name->id (memoize (fn [db-id table-name]
- {:pre [(integer? db-id)
- (keyword? table-name)]
- :post [(integer? %)]}
- (sel :one :id Table :db_id db-id :name (name table-name))))]
- (fn [table-name]
- {:pre [(integer? *db-id*)]}
- (-table-name->id *db-id* table-name))))
+;; (def ^{:arglists '([table-name])} table-name->id
+;; "Given keyword TABLE-NAME, fetch ID of corresponding `Table` in `*db*`."
+;; (let [-table-name->id (memoize (fn [db-id table-name]
+;; {:pre [(integer? db-id)
+;; (keyword? table-name)]
+;; :post [(integer? %)]}
+;; (sel :one :id Table :db_id db-id :name (name table-name))))]
+;; (fn [table-name]
+;; {:pre [(integer? *db-id*)]}
+;; (-table-name->id *db-id* table-name))))
;; ## Driver-Independent QP Tests
diff --git a/test/metabase/test/data.clj b/test/metabase/test/data.clj
new file mode 100644
index 00000000000..3db9abe24a3
--- /dev/null
+++ b/test/metabase/test/data.clj
@@ -0,0 +1,116 @@
+(ns metabase.test.data
+ "Eventual replacement for `metabase.test-data`.
+ Designed to handle multiple test data sets more nicely."
+ (:require [clojure.tools.logging :as log]
+ [colorize.core :as color]
+ [metabase.db :refer :all]
+ [metabase.driver :as driver]
+ (metabase.models [database :refer [Database]]
+ [org :refer [Org]])
+ (metabase.test.data [interface :as i]))
+ (:import metabase.test.data.h2.H2TestData
+ metabase.test.data.mongo.MongoTestData))
+
+;; ## Test Org
+
+(def ^:const ^:private test-org-name "Test Organization")
+
+(def test-org
+ (delay (or (sel :one Org :name test-org-name)
+ (ins Org
+ :name test-org-name
+ :slug "test"
+ :inherits true))))
+
+(def test-org-id
+ (delay (:id @test-org)))
+
+
+;; ## Test Data Instances
+
+(def h2 (H2TestData.))
+(def mongo (MongoTestData.))
+
+(def engine-name->test-data
+ {:h2 h2
+ :mongo mongo})
+
+(def ^:dynamic *engine*
+ :h2)
+
+(defn test-data []
+ (engine-name->test-data *engine*))
+
+(defmacro with-test-data [engine-name & body]
+ `(binding [*engine* ~engine-name]
+ (assert (test-data))
+ ~@body))
+
+
+;; ## Implementation-Agnostic Fns
+
+(defn load! []
+ (log/debug (color/cyan (format "Loading %s test data..." (name *engine*))))
+ (i/load! (test-data))
+ (let [db (ins Database
+ :engine *engine*
+ :organization_id @test-org-id
+ :name (db-name (test-data))
+ :details {:conn_str (connection-str (test-data))})]
+ (driver/sync-database! db)
+ (log/debug (color/green "Done."))
+ db))
+
+(defn db []
+ (let [db (or (sel :one Database :name (i/db-name (test-data)))
+ (load!))]
+ (assert (map? db))
+ db))
+
+(def db-id
+ (let [engine-name->db-id (memoize
+ (fn [engine-name]
+ {:post [(integer? engine-name)]}
+ (with-test-data engine-name
+ (:id (db)))))])
+ (fn []
+ (engine-name->db-id *engine*)))
+
+(defn destroy! []
+ (cascade-delete Database :name (db-name (test-data))))
+
+(defn table-name->table [table-name]
+ {:pre [(keyword? table-name)]
+ :post [(map? %)]}
+ (i/table-name->table (test-data) (db-id) table-name))
+
+(def table-name->id
+ (let [engine+table-name->id (memoize
+ (fn [engine-name table-name]
+ {:pre [(keyword? table-name)]
+ :post [(integer? %)]}
+ (with-test-data engine-name
+ (i/table-name->id (test-data) (db-id) table-name))))]
+ (fn [table-name]
+ (engine+table-name->id *engine* table-name))))
+
+(defn field-name->field [table-name field-name]
+ {:pre [(keyword? table-name)
+ (keyword? field-name)]
+ :post [(map? %)]}
+ (i/field-name->field (test-data) table-name field-name))
+
+(defn field-name->id [table-name field-name]
+ {:pre [(keyword? table-name)
+ (keyword? field-name)]
+ :post [(integer? %)]}
+ (i/field-name->id (test-data) table-name field-name))
+
+
+;; ## Fns that Run Across *All* Test Datasets
+
+(defn load-all! []
+ (doseq [[engine test-data] engine-name->test-data]
+
+ (i/load! test-data)
+ (log/debug (color/green "Ok."))))
diff --git a/test/metabase/test/data/h2.clj b/test/metabase/test/data/h2.clj
new file mode 100644
index 00000000000..3936f86ffef
--- /dev/null
+++ b/test/metabase/test/data/h2.clj
@@ -0,0 +1,11 @@
+(ns metabase.test.data.h2
+ (:require [metabase.test.data.interface :refer :all]))
+
+(deftype H2TestData []
+ TestData
+ (load! [_])
+ (db-name [_]
+ "Test Database")
+ (connection-str [_]
+ (let [filename (format "%s/target/test-data" (System/getProperty "user.dir"))]
+ (format "file:%s;AUTO_SERVER=TRUE;DB_CLOSE_DELAY=-1" filename))))
diff --git a/test/metabase/test/data/interface.clj b/test/metabase/test/data/interface.clj
new file mode 100644
index 00000000000..65563fff3cd
--- /dev/null
+++ b/test/metabase/test/data/interface.clj
@@ -0,0 +1,17 @@
+(ns metabase.test.data.interface
+ "`TestData` protocol.")
+
+(defprotocol TestData
+ ;; Loading
+ (load! [this]
+ "Load test data, if needed. This should create relevant data, and call `metabase.driver.sync-database!`.")
+
+ ;; DB
+ (db-name [this]
+ "Name to use for test `Database`.")
+ (connection-str [this]
+ "Connection string to use to connect to test `Database`.")
+
+ ;; Fetching Fns
+ (table-name->table [this db-id table-name])
+ (field-name->field [this table-name field-name]))
diff --git a/test/metabase/test/data/mongo.clj b/test/metabase/test/data/mongo.clj
new file mode 100644
index 00000000000..474ffc3663c
--- /dev/null
+++ b/test/metabase/test/data/mongo.clj
@@ -0,0 +1,18 @@
+(ns metabase.test.data.mongo
+ (:require [metabase.db :refer :all]
+ (metabase.models [table :refer [Table]])
+ [metabase.test.data.interface :refer :all]))
+
+(deftype MongoTestData []
+ TestData
+ (load! [_])
+
+ (db-name [_]
+ "Mongo Test")
+ (connection-str [_]
+ "mongodb://localhost/metabase-test")
+
+ (table-name->table [_ db-id table-name]
+ (sel :one Table :db_id db-id :name table-name))
+
+ (field-name->field [_ table-name field-name]))
diff --git a/test/metabase/test_setup.clj b/test/metabase/test_setup.clj
index 9c8526276f9..5edc65c8e2c 100644
--- a/test/metabase/test_setup.clj
+++ b/test/metabase/test_setup.clj
@@ -6,7 +6,8 @@
(metabase [core :as core]
[db :as db]
[task :as task]
- [test-data :refer :all])))
+ [test-data :as h2-test-data])
+ [metabase.driver.mongo.test-data :as mongo-test-data]))
(declare clear-test-db)
@@ -24,11 +25,17 @@
[]
(log/info "Starting up Metabase unit test runner")
;; clear out any previous test data that's lying around
+ (log/info "Clearing out test DB...")
(clear-test-db)
- ;; setup the db and migrate up to current schema
+ (log/info "Setting up test DB and running migrations...")
(db/setup-db :auto-migrate true)
- ;; this causes the test data to be loaded
- @test-db
+ (log/info "Loading test data: h2...")
+ @h2-test-data/test-db
+ (assert (integer? @h2-test-data/db-id))
+ (log/info "Loading test data: mongo...")
+ (mongo-test-data/destroy!)
+ @mongo-test-data/mongo-test-db
+ (assert (integer? @mongo-test-data/mongo-test-db-id))
;; startup test web server
(core/start-jetty)
;; start the task runner
--
GitLab