Skip to content
Snippets Groups Projects
Unverified Commit 08539ac1 authored by Cam Saul's avatar Cam Saul
Browse files

Merge branch 'release-0.32.0' into master :scream_cat:

parents 7a0e49f0 4480b24b
No related branches found
No related tags found
No related merge requests found
Showing
with 231 additions and 164 deletions
......@@ -405,7 +405,8 @@ function transformSingleSeries(s, series, seriesIndex) {
_transformed: true,
_seriesIndex: seriesIndex,
// use underlying column name as the seriesKey since it should be unique
_seriesKey: col ? col.name : name,
// EXCEPT for dashboard multiseries, so check seriesIndex == 0
_seriesKey: seriesIndex === 0 && col ? col.name : name,
},
data: {
rows: rows.map((row, rowIndex) => {
......
......@@ -149,17 +149,13 @@
[[lein-environ "1.1.0"]] ; easy access to environment variables
:env {:mb-run-mode "dev"}
:jvm-opts ["-Dlogfile.path=target/log"]
;; Log appender class needs to be compiled for log4j to use it. Same with the Quartz class load helper
:aot [metabase.logger
metabase.task.DynamicClassLoadHelper]}
:jvm-opts ["-Dlogfile.path=target/log"]}
:ci
{:jvm-opts ["-Xmx2500m"]}
:install
{:aot [metabase.logger
metabase.task.DynamicClassLoadHelper]}
{}
:install-for-building-drivers
{:auto-clean true
......
......@@ -14,8 +14,6 @@ log4j.appender.file.MaxBackupIndex=2
log4j.appender.file.layout=org.apache.log4j.PatternLayout
log4j.appender.file.layout.ConversionPattern=%d [%t] %-5p%c - %m%n
log4j.appender.metabase=metabase.logger.Appender
# customizations to logging by package
log4j.logger.metabase.driver=INFO
log4j.logger.metabase.plugins=DEBUG
......
......@@ -5,7 +5,6 @@ org.quartz.threadPool.threadCount = 10
# Don't phone home
org.quartz.scheduler.skipUpdateCheck: true
org.quartz.scheduler.classLoadHelper.class=metabase.task.DynamicClassLoadHelper
# Use the JDBC backend so we can cluster when running multiple instances!
# See http://www.quartz-scheduler.org/documentation/quartz-2.x/configuration/ConfigJDBCJobStoreClustering
......@@ -18,8 +17,6 @@ org.quartz.jobStore.dataSource=db
org.quartz.jobStore.isClustered = true
org.quartz.dataSource.db.validationQuery=SELECT 1
# By default, Quartz will fire triggers up to a minute late without considering them to be misfired; if it cannot fire
# anything within that period for one reason or another (such as all threads in the thread pool being tied up), the
# trigger is considered misfired. Threshold is in milliseconds.
......
......@@ -99,14 +99,14 @@
(api/defendpoint POST "/"
"Login."
[:as {{:keys [username password]} :body, remote-address :remote-addr}]
[:as {{:keys [username password]} :body, remote-address :remote-addr, :as request}]
{username su/NonBlankString
password su/NonBlankString}
(throttle-check (login-throttlers :ip-address) remote-address)
(throttle-check (login-throttlers :username) username)
(let [session-id (login username password)
response {:id session-id}]
(mw.session/set-session-cookie response session-id)))
(mw.session/set-session-cookie request response session-id)))
(api/defendpoint DELETE "/"
......@@ -164,7 +164,7 @@
(api/defendpoint POST "/reset_password"
"Reset password with a reset token."
[:as {{:keys [token password]} :body}]
[:as {{:keys [token password]} :body, :as request}]
{token su/NonBlankString
password su/ComplexPassword}
(or (when-let [{user-id :id, :as user} (valid-reset-token->user token)]
......@@ -176,6 +176,7 @@
;; after a successful password update go ahead and offer the client a new session that they can use
(let [session-id (create-session! user)]
(mw.session/set-session-cookie
request
{:success true
:session_id (str session-id)}
session-id)))
......@@ -253,7 +254,7 @@
(api/defendpoint POST "/google_auth"
"Login with Google Auth."
[:as {{:keys [token]} :body, remote-address :remote-addr}]
[:as {{:keys [token]} :body, remote-address :remote-addr, :as request}]
{token su/NonBlankString}
(throttle-check (login-throttlers :ip-address) remote-address)
;; Verify the token is valid with Google
......@@ -261,7 +262,7 @@
(log/info (trs "Successfully authenticated Google Auth token for: {0} {1}" given_name family_name))
(let [session-id (api/check-500 (google-auth-fetch-or-create-user! given_name family_name email))
response {:id session-id}]
(mw.session/set-session-cookie response session-id))))
(mw.session/set-session-cookie request response session-id))))
(api/define-routes)
......@@ -34,7 +34,8 @@
[:as {{:keys [token]
{:keys [name engine details is_full_sync is_on_demand schedules]} :database
{:keys [first_name last_name email password]} :user
{:keys [allow_tracking site_name]} :prefs} :body}]
{:keys [allow_tracking site_name]} :prefs} :body
:as request}]
{token SetupToken
site_name su/NonBlankString
first_name su/NonBlankString
......@@ -82,7 +83,7 @@
;; notify that we've got a new user in the system AND that this user logged in
(events/publish-event! :user-create {:user_id (:id new-user)})
(events/publish-event! :user-login {:user_id (:id new-user), :session_id (str session-id), :first_login true})
(mw.session/set-session-cookie {:id (str session-id)} session-id)))
(mw.session/set-session-cookie request {:id (str session-id)} session-id)))
(api/defendpoint POST "/validate"
......
......@@ -19,7 +19,7 @@
"Logs."
[]
(api/check-superuser)
(logger/get-messages))
(logger/messages))
(api/defendpoint GET "/stats"
"Anonymous usage stats. Endpoint for testing, and eventually exposing this to instance admins to let them see
......
......@@ -348,7 +348,7 @@
"initialPoolSize" 1
"maxPoolSize" 15})
(defn connection-pool
(defn- new-connection-pool
"Create a C3P0 connection pool for the given database `spec`."
[spec]
(connection-pool/connection-pool-spec spec application-db-connection-pool-properties))
......@@ -358,7 +358,7 @@
:postgres :ansi
:h2 :h2
:mysql :mysql))
(db/set-default-db-connection! (connection-pool spec)))
(db/set-default-db-connection! (new-connection-pool spec)))
;;; +----------------------------------------------------------------------------------------------------------------+
......
(ns metabase.logger
(:gen-class
:extends org.apache.log4j.AppenderSkeleton
:name metabase.logger.Appender)
(:require [amalloy.ring-buffer :refer [ring-buffer]]
[clj-time
[coerce :as coerce]
[core :as t]
[format :as time]]
[clojure.string :as str])
(:import org.apache.log4j.spi.LoggingEvent))
(:import [org.apache.log4j Appender AppenderSkeleton Logger]
org.apache.log4j.spi.LoggingEvent))
(def ^:private ^:const max-log-entries 2500)
(defonce ^:private messages (atom (ring-buffer max-log-entries)))
(defonce ^:private messages* (atom (ring-buffer max-log-entries)))
;; TODO - rename to `messages`
(defn get-messages
(defn messages
"Get the list of currently buffered log entries, from most-recent to oldest."
[]
(reverse (seq @messages)))
(reverse (seq @messages*)))
(defonce ^:private formatter (time/formatter "MMM dd HH:mm:ss" (t/default-time-zone)))
......@@ -36,22 +32,20 @@
(format "%s \033[1m%s %s\033[0m :: %s" ts level fqns msg))
(seq (.getThrowableStrRep event)))))
(defn -append
"Append a new EVENT to the `messages` atom.
[Overrides an `AppenderSkeleton`
method](http://logging.apache.org/log4j/1.2/apidocs/org/apache/log4j/AppenderSkeleton.html#append(org.apache.log4j.spi.LoggingEvent))"
[_, ^LoggingEvent event]
(swap! messages conj (event->log-string event))
nil)
(defn -close
"No-op if something tries to close this logging appender.
[Overrides an `Appender` method](http://logging.apache.org/log4j/1.2/apidocs/org/apache/log4j/Appender.html#close())"
[_]
nil)
(defn -requiresLayout
"The MB logger doesn't require a layout.
[Overrides an `Appender` method](http://logging.apache.org/log4j/1.2/apidocs/org/apache/log4j/Appender.html#getLayout())"
[_]
false)
(defn- metabase-appender ^Appender []
(proxy [AppenderSkeleton] []
(append [event]
(swap! messages* conj (event->log-string event))
nil)
(close []
nil)
(requiresLayout []
false)))
(defonce ^:private has-added-appender? (atom false))
(when-not *compile-files*
(when-not @has-added-appender?
(reset! has-added-appender? true)
(.addAppender (Logger/getRootLogger) (metabase-appender))))
(ns metabase.middleware.session
"Ring middleware related to session (binding current user and permissions)."
(:require [metabase
(:require [clojure.string :as str]
[metabase
[config :as config]
[db :as mdb]
[public-settings :as public-settings]]
[metabase.api.common :refer [*current-user* *current-user-id* *current-user-permissions-set*
*is-superuser?*]]
[db :as mdb]]
[metabase.api.common :refer [*current-user* *current-user-id* *current-user-permissions-set* *is-superuser?*]]
[metabase.core.initialization-status :as init-status]
[metabase.models
[session :refer [Session]]
......@@ -13,8 +12,7 @@
[ring.util.response :as resp]
[schema.core :as s]
[toucan.db :as db])
(:import java.net.URL
java.util.UUID
(:import java.util.UUID
org.joda.time.DateTime))
;; How do authenticated API requests work? Metabase first looks for a cookie called `metabase.SESSION`. This is the
......@@ -44,9 +42,38 @@
response
{:body response, :status 200}))
(defn- https-request?
"True if the original request made by the frontend client (i.e., browser) was made over HTTPS.
In many production instances, a reverse proxy such as an ELB or nginx will handle SSL termination, and the actual
request handled by Jetty will be over HTTP."
[{{:strs [x-forwarded-proto x-forwarded-protocol x-url-scheme x-forwarded-ssl front-end-https origin]} :headers
:keys [scheme]}]
(cond
;; If `X-Forwarded-Proto` is present use that. There are several alternate headers that mean the same thing. See
;; https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Forwarded-Proto
(or x-forwarded-proto x-forwarded-protocol x-url-scheme)
(= "https" (str/lower-case (or x-forwarded-proto x-forwarded-protocol x-url-scheme)))
;; If none of those headers are present, look for presence of `X-Forwarded-Ssl` or `Frontend-End-Https`, which
;; will be set to `on` if the original request was over HTTPS.
(or x-forwarded-ssl front-end-https)
(= "on" (str/lower-case (or x-forwarded-ssl front-end-https)))
;; If none of the above are present, we are most not likely being accessed over a reverse proxy. Still, there's a
;; good chance `Origin` will be present because it should be sent with `POST` requests, and most auth requests are
;; `POST`. See https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Origin
origin
(str/starts-with? (str/lower-case origin) "https")
;; Last but not least, if none of the above are set (meaning there are no proxy servers such as ELBs or nginx in
;; front of us), we can look directly at the scheme of the request sent to Jetty.
scheme
(= scheme :https)))
(s/defn set-session-cookie
"Add a `Set-Cookie` header to `response` to persist the Metabase session."
[response, session-id :- UUID]
[request response, session-id :- UUID]
(-> response
wrap-body-if-needed
(clear-cookie metabase-legacy-session-cookie)
......@@ -58,9 +85,9 @@
:http-only true
:path "/api"
:max-age (config/config-int :max-session-age)}
;; If Metabase is running over HTTPS (hopefully always except for local dev instances) then make sure to
;; make this cookie HTTPS-only
(when (some-> (public-settings/site-url) URL. .getProtocol (= "https"))
;; If the authentication request request was made over HTTPS (hopefully always except for local dev instances)
;; add `Secure` attribute so the cookie is only sent over HTTPS.
(when (https-request? request)
{:secure true})))))
(defn clear-session-cookie
......
......@@ -75,7 +75,7 @@
" "
(.getMessage e))})))
fetch-token-status-timeout-ms
{:valid false, :status (tru "Token validation timed out.")}))
{:valid false, :status (str (tru "Token validation timed out."))}))
(def ^:private ^{:arglists '([token])} fetch-token-status
"TTL-memoized version of `fetch-token-status*`. Caches API responses for 5 minutes. This is important to avoid making
......
......@@ -7,14 +7,17 @@
function which accepts zero arguments. This function is dynamically resolved and called exactly once when the
application goes through normal startup procedures. Inside this function you can do any work needed and add your
task to the scheduler as usual via `schedule-task!`."
(:require [clojure.string :as str]
(:require [clojure.java.jdbc :as jdbc]
[clojure.string :as str]
[clojure.tools.logging :as log]
[clojurewerkz.quartzite.scheduler :as qs]
[metabase
[db :as mdb]
[util :as u]]
[metabase.plugins.classloader :as classloader]
[metabase.util.i18n :refer [trs]]
[schema.core :as s])
[schema.core :as s]
[toucan.db :as db])
(:import [org.quartz JobDetail JobKey Scheduler Trigger TriggerKey]))
;;; +----------------------------------------------------------------------------------------------------------------+
......@@ -24,22 +27,6 @@
(defonce ^:private quartz-scheduler
(atom nil))
;; whenever the value of `quartz-scheduler` changes:
;;
;; 1. shut down the old scheduler, if there was one
;; 2. start the new scheduler, if there is one
(add-watch
quartz-scheduler
::quartz-scheduler-watcher
(fn [_ _ old-scheduler new-scheduler]
(when-not (identical? old-scheduler new-scheduler)
(when old-scheduler
(log/debug (trs "Stopping Quartz Scheduler {0}" old-scheduler))
(qs/shutdown old-scheduler))
(when new-scheduler
(log/debug (trs "Starting Quartz Scheduler {0}" new-scheduler))
(qs/start new-scheduler)))))
(defn- scheduler
"Fetch the instance of our Quartz scheduler. Call this function rather than dereffing the atom directly because there
are a few places (e.g., in tests) where we swap the instance out."
......@@ -88,6 +75,70 @@
(log/error e (trs "Error initializing task {0}" k))))))
;;; +----------------------------------------------------------------------------------------------------------------+
;;; | Quartz Scheduler Connection Provider |
;;; +----------------------------------------------------------------------------------------------------------------+
;; Custom `ConnectionProvider` implementation that uses our application DB connection pool to provide connections.
(defrecord ^:private ConnectionProvider []
org.quartz.utils.ConnectionProvider
(getConnection [_]
;; get a connection from our application DB connection pool. Quartz will close it (i.e., return it to the pool)
;; when it's done
(jdbc/get-connection (db/connection)))
(shutdown [_]))
(when-not *compile-files*
(System/setProperty "org.quartz.dataSource.db.connectionProvider.class" (.getName ConnectionProvider)))
;;; +----------------------------------------------------------------------------------------------------------------+
;;; | Quartz Scheduler Class Load Helper |
;;; +----------------------------------------------------------------------------------------------------------------+
;; Custom `ClassLoadHelper` implementation that makes sure to require the namespaces that tasks live in (to make sure
;; record types are loaded) and that uses our canonical ClassLoader.
(defn- task-class-name->namespace-str
"Determine the namespace we need to load for one of our tasks.
(task-class-name->namespace-str \"metabase.task.upgrade_checks.CheckForNewVersions\")
;; -> \"metabase.task.upgrade-checks\""
[class-name]
(-> class-name
(str/replace \_ \-)
(str/replace #"\.\w+$" "")))
(defn- require-task-namespace
"Since Metabase tasks are defined in Clojure-land we need to make sure we `require` the namespaces where they are
defined before we try to load the task classes."
[class-name]
;; call `the-classloader` to force side-effects of making it the current thread context classloader
(classloader/the-classloader)
;; only try to `require` metabase.task classes; don't do this for other stuff that gets shuffled thru here like
;; Quartz classes
(when (str/starts-with? class-name "metabase.task.")
(require (symbol (task-class-name->namespace-str class-name)))))
(defn- load-class ^Class [^String class-name]
(require-task-namespace class-name)
(.loadClass (classloader/the-classloader) class-name))
(defrecord ^:private ClassLoadHelper []
org.quartz.spi.ClassLoadHelper
(initialize [_])
(getClassLoader [_]
(classloader/the-classloader))
(loadClass [_ class-name]
(load-class class-name))
(loadClass [_ class-name _]
(load-class class-name)))
(when-not *compile-files*
(System/setProperty "org.quartz.scheduler.classLoadHelper.class" (.getName ClassLoadHelper)))
;;; +----------------------------------------------------------------------------------------------------------------+
;;; | STARTING/STOPPING SCHEDULER |
;;; +----------------------------------------------------------------------------------------------------------------+
......@@ -97,38 +148,25 @@
connection properties ahead of time, we'll need to set these at runtime rather than Setting them in the
`quartz.properties` file.)"
[]
(let [{:keys [classname user password subname subprotocol type]} (mdb/jdbc-details)]
;; If we're using a Postgres application DB the driverDelegateClass has to be the Postgres-specific one rather
;; than the Standard JDBC one we define in `quartz.properties`
(when (= type :postgres)
(System/setProperty "org.quartz.jobStore.driverDelegateClass" "org.quartz.impl.jdbcjobstore.PostgreSQLDelegate"))
;; set other properties like URL, user, and password so Quartz knows how to connect
(doseq [[k, ^String v] {:driver classname
:URL (format "jdbc:%s:%s" subprotocol subname)
:user user
:password password}]
(when v
(System/setProperty (str "org.quartz.dataSource.db." (name k)) v)))))
(def ^:private start-scheduler-lock (Object.))
(when (= (mdb/db-type) :postgres)
(System/setProperty "org.quartz.jobStore.driverDelegateClass" "org.quartz.impl.jdbcjobstore.PostgreSQLDelegate")))
(defn start-scheduler!
"Start our Quartzite scheduler which allows jobs to be submitted and triggers to begin executing."
[]
(when-not @quartz-scheduler
(locking start-scheduler-lock
(when-not @quartz-scheduler
(set-jdbc-backend-properties!)
;; keep a reference to our scheduler
(reset! quartz-scheduler (qs/initialize))
;; look for job/trigger definitions
(set-jdbc-backend-properties!)
(let [new-scheduler (qs/initialize)]
(when (compare-and-set! quartz-scheduler nil new-scheduler)
(qs/start new-scheduler)
(find-and-load-tasks!)))))
(defn stop-scheduler!
"Stop our Quartzite scheduler and shutdown any running executions."
[]
;; setting `quartz-scheduler` to nil will cause it to shut down via the watcher on it
(reset! quartz-scheduler nil))
(let [[old-scheduler] (reset-vals! quartz-scheduler nil)]
(when old-scheduler
(qs/shutdown old-scheduler))))
;;; +----------------------------------------------------------------------------------------------------------------+
......
(ns metabase.task.DynamicClassLoadHelper
"This is needed to get the JDBC backend for Quartz working, or something like that. See
http://clojurequartz.info/articles/durable_quartz_stores.html for details."
(:gen-class
:extends clojure.lang.DynamicClassLoader
:exposes-methods {loadClass superLoadClass}
:implements [org.quartz.spi.ClassLoadHelper])
(:require [clojure.string :as str]))
;; docstrings are copies of the ones for the corresponding methods of the ClassLoadHelper interface
(defn -initialize
"void initialize()
Called to give the ClassLoadHelper a chance to initialize itself, including the opportunity to \"steal\" the class
loader off of the calling thread, which is the thread that is initializing Quartz."
[_])
(defn- task-class-name->namespace-str
"Determine the namespace we need to load for one of our tasks.
(task-class-name->namespace-str \"metabase.task.upgrade_checks.CheckForNewVersions\")
;; -> \"metabase.task.upgrade-checks\""
[class-name]
(-> class-name
(str/replace \_ \-)
(str/replace #"\.\w+$" "")))
(defn- require-task-namespace
"Since Metabase tasks are defined in Clojure-land we need to make sure we `require` the namespaces where they are
defined before we try to load the task classes."
[class-name]
;; only try to `require` metabase.task classes; don't do this for other stuff that gets shuffled thru here like
;; Quartz classes
(when (str/starts-with? class-name "metabase.task.")
(require (symbol (task-class-name->namespace-str class-name)))))
(defn -loadClass
"Class loadClass(String className)
Return the class with the given name."
([^metabase.task.DynamicClassLoadHelper this, ^String class-name]
(require-task-namespace class-name)
(.superLoadClass this class-name true)) ; loadClass(String name, boolean resolve)
([^metabase.task.DynamicClassLoadHelper this, ^String class-name, _]
(require-task-namespace class-name)
(.superLoadClass this class-name true)))
(defn -getClassLoader
"ClassLoader getClassLoader()
Enable sharing of the class-loader with 3rd party"
[this]
this)
(ns metabase.query-processor.middleware.parameters.dates-test
(:require [expectations :refer [expect]]
[metabase.query-processor.middleware.parameters.dates :as dates]))
;; year and month
(expect
{:end "2019-04-30", :start "2019-04-01"}
(dates/date-string->range "2019-04" "UTC"))
(expect
[:between
[:datetime-field [:field-literal "field" :type/DateTime] :day]
"2019-04-01"
"2019-04-30"]
(dates/date-string->filter "2019-04" [:field-literal "field" :type/DateTime]))
;; quarter year
(expect
{:start "2019-04-01", :end "2019-06-30"}
(dates/date-string->range "Q2-2019" "UTC"))
(expect
[:between
[:datetime-field [:field-literal "field" :type/DateTime] :day]
"2019-04-01"
"2019-06-30"]
(dates/date-string->filter "Q2-2019" [:field-literal "field" :type/DateTime]))
;; single day
(expect
{:start "2019-04-01", :end "2019-04-01"}
(dates/date-string->range "2019-04-01" "UTC"))
(expect
[:=
[:datetime-field [:field-literal "field" :type/DateTime] :day]
"2019-04-01"]
(dates/date-string->filter "2019-04-01" [:field-literal "field" :type/DateTime]))
;; day range
(expect
{:start "2019-04-01", :end "2019-04-03"}
(dates/date-string->range "2019-04-01~2019-04-03" "UTC"))
(expect
[:between
[:datetime-field [:field-literal "field" :type/DateTime] :day]
"2019-04-01"
"2019-04-03"]
(dates/date-string->filter "2019-04-01~2019-04-03" [:field-literal "field" :type/DateTime]))
;; after day
(expect
{:start "2019-04-01"}
(dates/date-string->range "2019-04-01~" "UTC"))
(expect
[:>
[:datetime-field [:field-literal "field" :type/DateTime] :day]
"2019-04-01"]
(dates/date-string->filter "2019-04-01~" [:field-literal "field" :type/DateTime]))
(ns metabase.task.DynamicClassLoadHelper-test
(:require [expectations :refer :all]
[metabase.task.DynamicClassLoadHelper :as DynamicClassLoadHelper]))
(expect
"metabase.task.upgrade-checks"
(#'DynamicClassLoadHelper/task-class-name->namespace-str "metabase.task.upgrade_checks.CheckForNewVersions"))
......@@ -3,7 +3,7 @@
testing is part of `metabase.models.database`, so there's an argument to be made that these sorts of tests could
just as easily belong to a `database-test` namespace."
(:require [clojure.string :as str]
[expectations :refer :all]
[expectations :refer [expect]]
[metabase.models.database :refer [Database]]
[metabase.task.sync-databases :as sync-db]
[metabase.test.util :as tu]
......@@ -14,6 +14,13 @@
[toucan.util.test :as tt])
(:import [metabase.task.sync_databases SyncAndAnalyzeDatabase UpdateFieldValues]))
;; make sure our annotations are present
(expect
(.isAnnotationPresent SyncAndAnalyzeDatabase org.quartz.DisallowConcurrentExecution))
(expect
(.isAnnotationPresent UpdateFieldValues org.quartz.DisallowConcurrentExecution))
(defn- replace-trailing-id-with-<id> [s]
(str/replace s #"\d+$" "<id>"))
......
(ns metabase.task-test
(:require [expectations :refer [expect]]
[metabase.task :as task]))
(expect
"metabase.task.upgrade-checks"
(#'task/task-class-name->namespace-str "metabase.task.upgrade_checks.CheckForNewVersions"))
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment