Skip to content
Snippets Groups Projects
Unverified Commit 6a8284bc authored by Cam Saul's avatar Cam Saul Committed by GitHub
Browse files

Fix (most of the) driver tests after merging PR to switch to clojure.test (#10882)

parent 9724189c
No related branches found
No related tags found
No related merge requests found
Showing
with 207 additions and 148 deletions
(defproject metabase/oracle-driver "1.0.0"
:min-lein-version "2.5.0"
:include-drivers-dependencies [#"^ojdbc[78]\.jar$"]
:include-drivers-dependencies [#"^ojdbc\d+\.jar$"]
:profiles
{:provided
......
......@@ -196,6 +196,15 @@
(def ^:private short-timezone-name (memoize short-timezone-name*))
(defn- resolve-setting [ns-symb setting-symb]
(classloader/require ns-symb)
(let [varr (or (ns-resolve ns-symb setting-symb)
(throw (Exception. (tru "Could not resolve Setting {0}/{1}" ns-symb setting-symb))))
f (var-get varr)]
(assert (ifn? f)
(tru "Invalid Setting: {0)/{1}" ns-symb setting-symb))
(f)))
;; TODO - it seems like it would be a nice performance win to cache this a little bit
(defn public-settings
"Return a simple map of key/value pairs which represent the public settings (`MetabaseBootstrap`) for the front-end
......@@ -203,35 +212,31 @@
[]
{:admin_email (admin-email)
:anon_tracking_enabled (anon-tracking-enabled)
:custom_geojson (setting/get :custom-geojson)
:custom_formatting (setting/get :custom-formatting)
:email_configured (do (classloader/require 'metabase.email)
((resolve 'metabase.email/email-configured?)))
:available_locales (available-locales-with-names)
:custom_formatting (custom-formatting)
:custom_geojson (resolve-setting 'metabase.api.geojson 'custom-geojson)
:email_configured (resolve-setting 'metabase.email 'email-configured?)
:embedding (enable-embedding)
:enable_query_caching (enable-query-caching)
:enable_nested_queries (enable-nested-queries)
:enable_query_caching (enable-query-caching)
:enable_xrays (enable-xrays)
:engines (driver.u/available-drivers-info)
:entities (types/types->parents :entity/*)
:ga_code "UA-60817802-1"
:google_auth_client_id (setting/get :google-auth-client-id)
:google_auth_client_id (resolve-setting 'metabase.api.session 'google-auth-client-id)
:has_sample_dataset (db/exists? 'Database, :is_sample true)
:hide_embed_branding (metastore/hide-embed-branding?)
:ldap_configured (do (classloader/require 'metabase.integrations.ldap)
((resolve 'metabase.integrations.ldap/ldap-configured?)))
:available_locales (available-locales-with-names)
:ldap_configured (resolve-setting 'metabase.integrations.ldap 'ldap-configured?)
:map_tile_server_url (map-tile-server-url)
:metastore_url metastore/store-url
:password_complexity password/active-password-complexity
:premium_token (metastore/premium-embedding-token)
:public_sharing (enable-public-sharing)
:report_timezone (setting/get :report-timezone)
:setup_token (do
(classloader/require 'metabase.setup)
((resolve 'metabase.setup/token-value)))
:report_timezone (resolve-setting 'metabase.driver 'report-timezone)
:setup_token (resolve-setting 'metabase.setup 'token-value)
:site_name (site-name)
:site_url (site-url)
:timezone_short (short-timezone-name (setting/get :report-timezone))
:timezones common/timezones
:types (types/types->parents :type/*)
:entities (types/types->parents :entity/*)
:version config/mb-version-info})
......@@ -123,6 +123,8 @@
`(expect ::truthy ~actual))
([expected actual]
`(t/deftest ~(symbol (format "expect-%d" (hash &form)))
(t/is
(~'expect= ~expected ~actual)))))
(let [test-name (symbol (format "expect-%d" (hash &form)))]
`(t/deftest ~test-name
(t/testing (format ~(str (name (ns-name *ns*)) ":%d") (:line (meta #'~test-name)))
(t/is
(~'expect= ~expected ~actual)))))))
(ns metabase.api.database-test
"Tests for /api/database endpoints."
(:require [clojure.string :as str]
(:require [clojure
[string :as str]
[test :refer :all]]
[expectations :refer [expect]]
[medley.core :as m]
[metabase
......@@ -24,6 +26,7 @@
[sync-metadata :as sync-metadata]]
[metabase.test
[data :as data]
[fixtures :as fixtures]
[util :as tu]]
[metabase.test.data
[env :as tx.env]
......@@ -34,6 +37,8 @@
[toucan.db :as db]
[toucan.util.test :as tt]))
(use-fixtures :once (fixtures/initialize :plugins))
;; HELPER FNS
(driver/register! ::test-driver
......@@ -204,7 +209,8 @@
(defn- all-test-data-databases []
(for [driver (conj tx.env/test-drivers :h2)
;; GA has no test extensions impl and thus data/db doesn't work with it
:when (not= driver :googleanalytics)]
;; Also it doesn't work for Druid either because DB must be flattened
:when (not= driver :googleanalytics :druid)]
(merge
default-db-details
(select-keys (driver/with-driver driver (data/db)) [:created_at :id :updated_at :timezone])
......
......@@ -2,6 +2,7 @@
"Tests for /api/session"
(:require [cheshire.core :as json]
[clj-http.client :as http]
[clojure.test :refer :all]
[expectations :refer [expect]]
[metabase
[email-test :as et]
......@@ -45,18 +46,24 @@
(client :post 400 "session" (-> (test-users/user->credentials :rasta)
(assoc :password "something else"))))
;; Test that people get blocked from attempting to login if they try too many times (Check that throttling works at
;; the API level -- more tests in the throttle library itself: https://github.com/metabase/throttle)
(expect
[{:errors {:username "Too many attempts! You must wait 15 seconds before trying again."}}
{:errors {:username "Too many attempts! You must wait 42 seconds before trying again."}}]
(let [login #(client :post 400 "session" {:username "fakeaccount3000@metabase.com", :password "toucans"})]
;; attempt to log in 10 times
(dorun (repeatedly 10 login))
;; throttling should now be triggered
[(login)
;; Trying to login immediately again should still return throttling error
(login)]))
;;
;;
(deftest login-throttling-test
(testing (str "Test that people get blocked from attempting to login if they try too many times (Check that"
" throttling works at the API level -- more tests in the throttle library itself:"
" https://github.com/metabase/throttle)")
(let [login (fn []
(-> (client :post 400 "session" {:username "fakeaccount3000@metabase.com", :password "toucans"})
:errors
:username))]
;; attempt to log in 10 times
(dorun (repeatedly 10 login))
(is (re= #"^Too many attempts! You must wait 1\d seconds before trying again\.$"
(login))
"throttling should now be triggered")
(is (re= #"^Too many attempts! You must wait 4\d seconds before trying again\.$"
(login))
"Trying to login immediately again should still return throttling error"))))
(defn- send-login-request [username & [{:or {} :as headers}]]
(try
......@@ -73,36 +80,33 @@
clean-key (fn [m k] (assoc-in m [k :attempts] (atom '())))]
(reduce clean-key throttlers ks)))
;; Test that source based throttling kicks in after the login failure threshold (50) has been reached
(expect
["Too many attempts! You must wait 15 seconds before trying again."
"Too many attempts! You must wait 42 seconds before trying again."]
(with-redefs [session-api/login-throttlers (cleaned-throttlers #'session-api/login-throttlers
[:username :ip-address])
public-settings/source-address-header (constantly "x-forwarded-for")]
(do
;;
(deftest failure-threshold-throttling-test
(testing "Test that source based throttling kicks in after the login failure threshold (50) has been reached"
(with-redefs [session-api/login-throttlers (cleaned-throttlers #'session-api/login-throttlers
[:username :ip-address])
public-settings/source-address-header (constantly "x-forwarded-for")]
(dotimes [n 50]
(let [response (send-login-request (format "user-%d" n)
{"x-forwarded-for" "10.1.2.3"})
status-code (:status response)]
(assert (= status-code 400) (str "Unexpected response status code:" status-code))))
[(-> (send-login-request "last-user" {"x-forwarded-for" "10.1.2.3"})
:body
json/parse-string
(get-in ["errors" "username"]))
(-> (send-login-request "last-user" {"x-forwarded-for" "10.1.2.3"})
:body
json/parse-string
(get-in ["errors" "username"]))])))
;; The same as above, but ensure that throttling is done on a per request source basis.
(expect
["Too many attempts! You must wait 15 seconds before trying again."
"Too many attempts! You must wait 42 seconds before trying again."]
(with-redefs [session-api/login-throttlers (cleaned-throttlers #'session-api/login-throttlers
[:username :ip-address])
public-settings/source-address-header (constantly "x-forwarded-for")]
(do
(let [error (fn []
(-> (send-login-request "last-user" {"x-forwarded-for" "10.1.2.3"})
:body
json/parse-string
(get-in ["errors" "username"])))]
(is (re= #"^Too many attempts! You must wait 1\d seconds before trying again\.$"
(error)))
(is (re= #"^Too many attempts! You must wait 4\d seconds before trying again\.$"
(error)))))))
;;
(deftest failure-threshold-per-request-source
(testing "The same as above, but ensure that throttling is done on a per request source basis."
(with-redefs [session-api/login-throttlers (cleaned-throttlers #'session-api/login-throttlers
[:username :ip-address])
public-settings/source-address-header (constantly "x-forwarded-for")]
(dotimes [n 50]
(let [response (send-login-request (format "user-%d" n)
{"x-forwarded-for" "10.1.2.3"})
......@@ -112,14 +116,15 @@
(let [response (send-login-request (format "round2-user-%d" n)) ; no x-forwarded-for
status-code (:status response)]
(assert (= status-code 400) (str "Unexpected response status code:" status-code))))
[(-> (send-login-request "last-user" {"x-forwarded-for" "10.1.2.3"})
:body
json/parse-string
(get-in ["errors" "username"]))
(-> (send-login-request "last-user" {"x-forwarded-for" "10.1.2.3"})
:body
json/parse-string
(get-in ["errors" "username"]))])))
(let [error (fn []
(-> (send-login-request "last-user" {"x-forwarded-for" "10.1.2.3"})
:body
json/parse-string
(get-in ["errors" "username"])))]
(is (re= #"^Too many attempts! You must wait 1\d seconds before trying again\.$"
(error)))
(is (re= #"^Too many attempts! You must wait 4\d seconds before trying again\.$"
(error)))))))
;; ## DELETE /api/session
......@@ -166,20 +171,20 @@
(defn- send-password-reset [& [expected-status & more]]
(client :post (or expected-status 200) "session/forgot_password" {:email "not-found@metabase.com"}))
(expect
["Too many attempts! You must wait 15 seconds before trying again."
"Too many attempts! You must wait 15 seconds before trying again."] ; `throttling/check` gives 15 in stead of 42
(deftest forgot-password-throttling-test
(with-redefs [session-api/forgot-password-throttlers (cleaned-throttlers #'session-api/forgot-password-throttlers
[:email :ip-address])]
(do
(dotimes [n 10]
(send-password-reset))
[(-> (send-password-reset 400)
:errors
:email)
(-> (send-password-reset 400)
:errors
:email)])))
(dotimes [n 10]
(send-password-reset))
(let [error (fn []
(-> (send-password-reset 400)
:errors
:email))]
(is (= "Too many attempts! You must wait 15 seconds before trying again."
(error)))
;;`throttling/check` gives 15 in stead of 42
(is (= "Too many attempts! You must wait 15 seconds before trying again."
(error))))))
;; POST /api/session/reset_password
......@@ -283,24 +288,31 @@
;;; ------------------------------------------ TESTS FOR GOOGLE AUTH STUFF -------------------------------------------
;;; tests for email->domain
(expect "metabase.com" (#'session-api/email->domain "cam@metabase.com"))
(expect "metabase.co.uk" (#'session-api/email->domain "cam@metabase.co.uk"))
(expect "metabase.com" (#'session-api/email->domain "cam.saul+1@metabase.com"))
(deftest email->domain-test
(are [domain email] (is (= domain
(#'session-api/email->domain email))
(format "Domain of email address '%s'" email))
"metabase.com" "cam@metabase.com"
"metabase.co.uk" "cam@metabase.co.uk"
"metabase.com" "cam.saul+1@metabase.com"))
;;; tests for email-in-domain?
(expect true (#'session-api/email-in-domain? "cam@metabase.com" "metabase.com"))
(expect false (#'session-api/email-in-domain? "cam.saul+1@metabase.co.uk" "metabase.com"))
(expect true (#'session-api/email-in-domain? "cam.saul+1@metabase.com" "metabase.com"))
(deftest email-in-domain-test
(are [in-domain? email domain] (is (= in-domain?
(#'session-api/email-in-domain? email domain))
(format "Is email '%s' in domain '%s'?" email domain))
true "cam@metabase.com" "metabase.com"
false "cam.saul+1@metabase.co.uk" "metabase.com"
true "cam.saul+1@metabase.com" "metabase.com"))
;;; tests for autocreate-user-allowed-for-email?
(expect
(tu/with-temporary-setting-values [google-auth-auto-create-accounts-domain "metabase.com"]
(#'session-api/autocreate-user-allowed-for-email? "cam@metabase.com")))
(expect
false
(deftest allow-autocreation-test
(tu/with-temporary-setting-values [google-auth-auto-create-accounts-domain "metabase.com"]
(#'session-api/autocreate-user-allowed-for-email? "cam@expa.com")))
(are [allowed? email] (is (= allowed?
(#'session-api/autocreate-user-allowed-for-email? email))
(format "Can we autocreate an account for email '%s'?" email))
true "cam@metabase.com"
false "cam@expa.com")))
;;; tests for google-auth-create-new-user!
......
(ns metabase.public-settings-test
(:require [environ.core :as env]
(:require [clojure.test :refer :all]
[environ.core :as env]
[expectations :refer [expect]]
[metabase.models.setting :as setting]
[metabase.public-settings :as public-settings]
[metabase.test.util :as tu]
[metabase.test
[fixtures :as fixtures]
[util :as tu]]
[metabase.test.util.log :as tu.log]
[metabase.util.i18n :refer [tru]]
[puppetlabs.i18n.core :as i18n]))
(use-fixtures :once (fixtures/initialize :db))
;; double-check that setting the `site-url` setting will automatically strip off trailing slashes
(expect
"http://localhost:3000"
......@@ -69,14 +74,17 @@
{:get-string (setting/get-string :site-url)
:site-url (public-settings/site-url)})))
;; if `site-url` is set via an env var, and it's invalid, we should return `nil` rather than having the whole instance break
(expect
(deftest invalid-site-url-env-var-test
{:get-string "asd_12w31%$;", :site-url nil}
(tu.log/suppress-output
(with-redefs [env/env (assoc env/env :mb-site-url "asd_12w31%$;")]
(tu/with-temporary-setting-values [site-url nil]
{:get-string (setting/get-string :site-url)
:site-url (public-settings/site-url)}))))
(testing (str "If `site-url` is set via an env var, and it's invalid, we should return `nil` rather than having the"
" whole instance break")
(tu.log/suppress-output
(with-redefs [env/env (assoc env/env :mb-site-url "asd_12w31%$;")]
(tu/with-temporary-setting-values [site-url nil]
(is (= "asd_12w31%$;"
(setting/get-string :site-url)))
(is (= nil
(public-settings/site-url))))))))
(expect
"HOST"
......
......@@ -60,9 +60,11 @@
[drivers expected actual]
;; Make functions to get expected/actual so the code is only compiled one time instead of for every single driver
;; speeds up loading of metabase.driver.query-processor-test significantly
`(t/deftest ~(symbol (str "expect-with-drivers-" (hash &form)))
(test-drivers ~drivers
(t/is (~'expect= ~expected ~actual)))))
(let [symb (symbol (str "expect-with-drivers-" (hash &form)))]
`(t/deftest ~symb
(t/testing (format ~(str (name (ns-name *ns*)) ":%d") (:line (meta #'~symb)))
(test-drivers ~drivers
(t/is (~'expect= ~expected ~actual)))))))
(defmacro test-all-drivers [& body]
`(test-drivers tx.env/test-drivers ~@body))
......
......@@ -9,28 +9,16 @@
# just test against :h2 (default)
DRIVERS=h2"
(:require [clojure.string :as str]
[clojure.tools.logging :as log]
(:require [clojure.tools.logging :as log]
[colorize.core :as color]
[environ.core :refer [env]]
[metabase.util :as u]))
(defn- get-drivers-from-env
"Return a set of drivers to test against from the env var `DRIVERS`."
[]
(when (seq (env :engines))
(println
(u/format-color 'red
"The env var ENGINES is no longer supported. Please specify drivers to run tests against with DRIVERS instead.")))
(when-let [env-drivers (some-> (env :drivers) str/lower-case)]
(set (for [engine (str/split env-drivers #",")
:when engine]
(keyword engine)))))
[metabase.test.data.env.impl :as impl]
[metabase.test.initialize :as initialize]))
(defonce ^{:doc (str "Set of names of drivers we should run tests against. By default, this only contains `:h2` but can"
" be overriden by setting env var `DRIVERS`.")}
test-drivers
(let [drivers (or (get-drivers-from-env)
#{:h2})]
(let [drivers (impl/get-test-drivers)]
(log/info (color/cyan "Running QP tests against these drivers: " drivers))
(when-not (= drivers #{:h2})
(initialize/initialize-if-needed! :plugins))
drivers))
(ns metabase.test.data.env.impl
(:require [clojure.string :as str]
[environ.core :refer [env]]
[metabase.util :as u]))
(defn- get-drivers-from-env []
(when (seq (env :engines))
(println
(u/format-color 'red
"The env var ENGINES is no longer supported. Please specify drivers to run tests against with DRIVERS instead.")))
(when-let [env-drivers (some-> (env :drivers) str/lower-case)]
(set (for [engine (str/split env-drivers #",")
:when engine]
(keyword engine)))))
(defn get-test-drivers
"Return a set of drivers to test against from the env var `DRIVERS`."
[]
(or (get-drivers-from-env) #{:h2}))
(ns metabase.test.data.users
"Code related to creating / managing fake `Users` for testing purposes."
(:require [medley.core :as m]
(:require [clojure.test :as t]
[medley.core :as m]
[metabase
[http-client :as http]
[util :as u]]
......@@ -164,4 +165,5 @@
`user-kwd`."
{:style/indent 1}
[user-kwd & body]
`(do-with-test-user ~user-kwd (fn [] ~@body)))
`(t/testing ~(format "with test user %s" user-kwd)
(do-with-test-user ~user-kwd (fn [] ~@body))))
(ns metabase.test.initialize
"Logic for initializing different components that needed to be initialized when running tests."
"Logic for initializing different components that need to be initialized when running tests."
(:require [clojure.string :as str]
[colorize.core :as colorize]))
......@@ -16,12 +16,17 @@
(doseq [k args]
(initialize-if-needed! k)))
(defn- task-name-init-message [task-name]
(defn- log-init-message [task-name]
(let [body (format "| Initializing %s... |" task-name)
border (str \+ (str/join (repeat (- (count body) 2) \-)) \+)]
(str "\n"
(str/join "\n" [border body border])
"\n")))
(println
(colorize/blue
(str "\n"
(str/join "\n" [border body border])
"\n")))
#_(println "REASON:")
#_(u/pprint-to-str 'blue (u/filtered-stacktrace (Thread/currentThread)))
#_"\n"))
(defmacro ^:private define-initialization [task-name & body]
(let [delay-symb (vary-meta (symbol (format "init-%s-%d" (name task-name) (hash &form)))
......@@ -29,7 +34,7 @@
`(do
(defonce ~delay-symb
(delay
(println (colorize/blue ~(task-name-init-message task-name)))
(log-init-message ~(keyword task-name))
~@body
nil))
(defmethod initialize-if-needed! ~(keyword task-name)
......@@ -40,10 +45,6 @@
(require 'metabase.test.initialize.plugins)
((resolve 'metabase.test.initialize.plugins/init!)))
(define-initialization :scheduler
(require 'metabase.test.initialize.scheduler)
((resolve 'metabase.test.initialize.scheduler/init!)))
(define-initialization :db
(require 'metabase.test.initialize.db)
((resolve 'metabase.test.initialize.db/init!)))
......
(ns metabase.test.initialize.db
(:require [metabase.db :as mdb]))
(:require [metabase
[db :as mdb]
[task :as task]]))
(defn init! []
(println (format "Setting up %s test DB and running migrations..." (mdb/db-type)))
(#'task/set-jdbc-backend-properties!)
(mdb/setup-db!))
......@@ -5,7 +5,7 @@
[plugins :as plugins]
[util :as u]]
[metabase.plugins.initialize :as plugins.init]
[metabase.test.data.env :as tx.env]
[metabase.test.data.env.impl :as tx.env.impl]
[yaml.core :as yaml]))
(defn- driver-plugin-manifest [driver]
......@@ -26,7 +26,7 @@
Work some magic and find manifest files and load them the way the plugins namespace would have done."
([]
(load-plugin-manifests! tx.env/test-drivers))
(load-plugin-manifests! (tx.env.impl/get-test-drivers)))
([drivers]
(doseq [driver drivers
......
(ns metabase.test.initialize.scheduler
(:require [metabase.task :as task]))
(defn init! []
;; we don't want to actually start the task scheduler (we don't want sync or other stuff happening in the BG
;; while running tests), but we still need to make sure it sets itself up properly so tasks can get scheduled
;; without throwing Exceptions
(#'task/set-jdbc-backend-properties!))
(ns metabase.test.redefs
(:require [toucan.util.test :as tt]))
(:require [clojure.test :as t]
[toucan.util.test :as tt]))
;; wrap `do-with-temp` so it initializes the DB before doing the other stuff it usually does
(when-not (::wrapped? (meta #'tt/do-with-temp))
......@@ -13,3 +14,15 @@
;; mark `expect-with-temp` as deprecated -- it's not needed for `deftest`-style tests
(alter-meta! #'tt/expect-with-temp assoc :deprecated true)
;; TODO - not a good long-term place to put this.
(defmethod t/assert-expr 're= [msg [_ pattern s]]
`(let [pattern# ~pattern
s# ~s
matches?# (when s#
(re-matches pattern# s#))]
(t/do-report
{:type (if matches?# :pass :fail)
:message ~msg
:expected pattern#
:actual s#})))
......@@ -33,7 +33,9 @@
[task-history :refer [TaskHistory]]
[user :refer [User]]]
[metabase.plugins.classloader :as classloader]
[metabase.test.data :as data]
[metabase.test
[data :as data]
[initialize :as initialize]]
[metabase.util.date :as du]
[schema.core :as s]
[toucan.db :as db]
......@@ -256,6 +258,7 @@
Prefer the macro `with-temporary-setting-values` over using this function directly."
{:style/indent 2}
[setting-k value f]
(initialize/initialize-if-needed! :db)
(let [setting (#'setting/resolve-setting setting-k)
original-value (when (or (#'setting/db-or-cache-value setting)
(#'setting/env-var-value setting))
......@@ -273,7 +276,8 @@
(with-temporary-setting-values [google-auth-auto-create-accounts-domain \"metabase.com\"]
(google-auth-auto-create-accounts-domain)) -> \"metabase.com\""
[[setting-k value & more] & body]
(let [body `(do-with-temporary-setting-value ~(keyword setting-k) ~value (fn [] ~@body))]
(let [body `(t/testing ~(format "Setting %s = %s" (keyword setting-k) value)
(do-with-temporary-setting-value ~(keyword setting-k) ~value (fn [] ~@body)))]
(if (seq more)
`(with-temporary-setting-values ~more ~body)
body)))
......
(ns metabase.test.util.timezone
(:require [clj-time.core :as time]
[metabase.driver :as driver]
[metabase.test.initialize :as initialize]
[metabase.util.date :as du])
(:import java.util.TimeZone
org.joda.time.DateTimeZone))
......@@ -19,7 +20,8 @@
(defn call-with-jvm-tz
"Invokes the thunk `F` with the JVM timezone set to `DTZ` (String or instance of TimeZone or DateTimeZone), puts the
various timezone settings back the way it found it when it exits."
[dtz f]
[dtz thunk]
(initialize/initialize-if-needed! :db)
(let [dtz (->datetimezone dtz)
orig-tz (TimeZone/getDefault)
orig-dtz (time/default-time-zone)
......@@ -35,7 +37,7 @@
;; We read the system property directly when formatting results, so this needs to be changed
(System/setProperty "user.timezone" (.getID dtz))
(with-redefs [du/jvm-timezone (delay (.toTimeZone dtz))]
(f))
(thunk))
(finally
;; We need to ensure we always put the timezones back the way
;; we found them as it will cause test failures
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment