Skip to content
Snippets Groups Projects
Commit 8103f7c8 authored by Cam Saül's avatar Cam Saül
Browse files

metabase/throttle is now a separate lib :yum:

parent 2cfc68a4
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
project.clj
+ 2
− 0
View file @ 8103f7c8
......@@ -48,6 +48,8 @@
com.sun.jdmk/jmxtools
com.sun.jmx/jmxri]]
[medley "0.7.2"] ; lightweight lib of useful functions
[medley "0.7.1"] ; lightweight lib of useful functions
[metabase/throttle "1.0.0"] ; Tools for throttling access to API endpoints and other code pathways
[mysql/mysql-connector-java "5.1.38"] ; MySQL JDBC driver
[net.sf.cssbox/cssbox "4.10"
:exclusions [org.slf4j/slf4j-api]]
......
src/metabase/api/common/throttle.clj deleted 100644 → 0
+ 0
− 134
View file @ 2cfc68a4
(ns metabase.api.common.throttle
(:require [clojure.math.numeric-tower :as math])
(:import (clojure.lang Atom
Keyword)))
;;; # THROTTLING
;;
;; A `Throttler` is a simple object used for throttling API endpoints. It keeps track of all calls to an API endpoint
;; with some value over some past period of time. If the number of calls with this value exceeds some threshold,
;; an exception is thrown, telling a user they must wait some period of time before trying again.
;;
;; ### EXAMPLE
;;
;; Let's consider the email throttling done by POST /api/session.
;; The basic concept here is to keep a list of failed logins over the last hour. This list looks like:
;;
;; (["cam@metabase.com" 1438045261132]
;; ["cam@metabase.com" 1438045260450]
;; ["cam@metabase.com" 1438045259037]
;; ["cam@metabase.com" 1438045258204])
;;
;; Every time there's a login attempt, push a new pair of [email timestamp (milliseconds)] to the front of the list.
;; The list is thus automatically ordered by date, and we can drop the portion of the list with logins that are over
;; an hour old as needed.
;;
;; Once a User has reached some number of login attempts over the past hour (e.g. 5), calculate some delay before
;; they're allowed to try to log in again (e.g., 15 seconds). This number will increase exponentially as the number of
;; recent failures increases (e.g., 40 seconds for 6 failed attempts, 90 for 7 failed attempts, etc).
;;
;; If applicable, calucate the time since the last failed attempt, and throw an exception telling the user the number
;; of seconds they must wait before trying again.
;;
;; ### USAGE
;;
;; Define a new throttler with `make-throttler`, overriding default settings as needed.
;;
;; (require '[metabase.api.common.throttle :as throttle])
;; (def email-throttler (throttle/make-throttler :email, :attempts-threshold 10))
;;
;; Then call `check` within the body of an endpoint with some value to apply throttling.
;;
;; (defendpoint POST [:as {{:keys [email]} :body}]
;; (throttle/check email-throttler email)
;; ...)
;;; # PUBLIC INTERFACE
(declare calculate-delay
remove-old-attempts)
(defrecord Throttler [;; Name of the API field/value being checked. Used to generate appropriate API error messages, so
;; they'll be displayed on the right part of the screen
^Keyword exception-field-key
;; [Internal] List of attempt entries. These are pairs of [key timestamp (ms)],
;; e.g. ["cam@metabase.com" 1438045261132]
^Atom attempts
;; Amount of time to keep an entry in ATTEMPTS before dropping it.
^Integer attempt-ttl-ms
;; Number of attempts allowed with a given key before throttling is applied.
^Integer attempts-threshold
;; Once throttling is in effect, initial delay before allowing another attempt. This grows
;; according to DELAY-EXPONENT.
^Integer initial-delay-ms
;; For each subsequent failure past ATTEMPTS-THRESHOLD, increase the delay to
;; INITIAL-DELAY-MS * (num-attempts-over-theshold ^ DELAY-EXPONENT). e.g. if INITIAL-DELAY-MS is 15
;; and DELAY-EXPONENT is 2, the first attempt past ATTEMPTS-THRESHOLD will require the user to wait
;; 15 seconds (15 * 1^2), the next attempt after that 60 seconds (15 * 2^2), then 135, and so on.
^Integer delay-exponent])
;; These are made private because you should use `make-throttler` instead.
(alter-meta! #'->Throttler assoc :private true)
(alter-meta! #'map->Throttler assoc :private true)
(def ^:private ^:const throttler-defaults
{:initial-delay-ms (* 15 1000)
:attempts-threshold 10
:delay-exponent 1.5
:attempt-ttl-ms (* 1000 60 60)})
(defn make-throttler
"Create a new `Throttler`.
(require '[metabase.api.common.throttle :as throttle])
(def email-throttler (throttle/make-throttler :email, :attempts-threshold 10))"
[exception-field-key & {:as kwargs}]
(map->Throttler (merge throttler-defaults kwargs {:attempts (atom '())
:exception-field-key exception-field-key})))
(defn check
"Throttle an API call based on values of KEYY. Each call to this function will record KEYY to THROTTLER's internal list;
if the number of entires containing KEYY exceed THROTTLER's thresholds, throw an exception.
(defendpoint POST [:as {{:keys [email]} :body}]
(throttle/check email-throttler email)
...)"
[^Throttler {:keys [attempts exception-field-key], :as throttler} keyy] ; technically, keyy can be nil so you can record *all* attempts
{:pre [(= (type throttler) Throttler)]}
(remove-old-attempts throttler)
(when-let [delay-ms (calculate-delay throttler keyy)]
(let [message (format "Too many attempts! You must wait %d seconds before trying again."
(int (math/round (/ delay-ms 1000))))]
(throw (ex-info message {:status-code 400
:errors {exception-field-key message}}))))
(swap! attempts conj [keyy (System/currentTimeMillis)]))
;;; # INTERNAL IMPLEMENTATION
(defn- remove-old-attempts
"Remove THROTTLER entires past the TTL."
[^Throttler {:keys [attempts attempt-ttl-ms]}]
(let [old-attempt-cutoff (- (System/currentTimeMillis) attempt-ttl-ms)
non-old-attempt? (fn [[_ timestamp]]
(> timestamp old-attempt-cutoff))]
(reset! attempts (take-while non-old-attempt? @attempts))))
(defn- calculate-delay
"Calculate the delay in milliseconds, if any, that should be applied to a given THROTTLER / KEYY combination."
([^Throttler throttler keyy]
(calculate-delay throttler keyy (System/currentTimeMillis)))
([^Throttler {:keys [attempts initial-delay-ms attempts-threshold delay-exponent]} keyy current-time-ms]
(let [[[_ most-recent-attempt-ms], :as keyy-attempts] (filter (fn [[k _]] (= k keyy)) @attempts)]
(when most-recent-attempt-ms
(let [num-recent-attempts (count keyy-attempts)
num-attempts-over-threshold (- (inc num-recent-attempts) attempts-threshold)] ; add one to the sum to account for the current attempt
(when (> num-attempts-over-threshold 0)
(let [delay-ms (* (math/expt num-attempts-over-threshold delay-exponent)
initial-delay-ms)
next-login-allowed-at (+ most-recent-attempt-ms delay-ms)
ms-till-next-login (- next-login-allowed-at current-time-ms)]
(when (> ms-till-next-login 0)
ms-till-next-login))))))))
src/metabase/api/session.clj
+ 1
− 1
View file @ 8103f7c8
......@@ -6,13 +6,13 @@
[hiccup.core :refer [html]]
[korma.core :as k]
[metabase.api.common :refer :all]
[metabase.api.common.throttle :as throttle]
[metabase.db :refer :all]
[metabase.email.messages :as email]
[metabase.events :as events]
(metabase.models [user :refer [User set-user-password set-user-password-reset-token]]
[session :refer [Session]]
[setting :as setting])
[metabase.throttle :as throttle]
[metabase.util.password :as pass]))
......
test/metabase/api/common/throttle_test.clj deleted 100644 → 0
+ 0
− 117
View file @ 2cfc68a4
(ns metabase.api.common.throttle-test
(:require [expectations :refer :all]
[metabase.api.common.throttle :as throttle]
[metabase.test.util :refer [resolve-private-fns]]))
(def ^:private test-throttler (throttle/make-throttler :test, :initial-delay-ms 5, :attempts-threshold 3, :delay-exponent 2, :attempt-ttl-ms 25))
;;; # tests for calculate-delay
(resolve-private-fns metabase.api.common.throttle calculate-delay)
;; no delay should be calculated for the 3rd attempt
(expect nil
(do (reset! (:attempts test-throttler) '([:x 100], [:x 99]))
(calculate-delay test-throttler :x 101)))
;; 4 ms delay on 4th attempt 1ms after the last
(expect 4
(do (reset! (:attempts test-throttler) '([:x 100], [:x 99], [:x 98]))
(calculate-delay test-throttler :x 101)))
;; 5 ms after last attempt, they should be allowed to try again
(expect nil
(do (reset! (:attempts test-throttler) '([:x 100], [:x 99], [:x 98]))
(calculate-delay test-throttler :x 105)))
;; However if this was instead the 5th attempt delay should grow exponentially (5 * 2^2 = 20), - 2ms = 18ms
(expect 18
(do (reset! (:attempts test-throttler) '([:x 100], [:x 99], [:x 98], [:x 97]))
(calculate-delay test-throttler :x 102)))
;; Should be allowed after 18 more secs
(expect nil
(do (reset! (:attempts test-throttler) '([:x 100], [:x 99], [:x 98], [:x 97]))
(calculate-delay test-throttler :x 120)))
;; Check that delay keeps growing according to delay-exponent (5 * 3^2 = 5 * 9 = 45)
(expect 45
(do (reset! (:attempts test-throttler) '([:x 108], [:x 100], [:x 99], [:x 98], [:x 97]))
(calculate-delay test-throttler :x 108)))
;;; # tests for check
(defn- attempt
([n]
(attempt n (gensym)))
([n k]
(let [attempt-once (fn []
(try
(Thread/sleep 1)
(throttle/check test-throttler k)
:success
(catch Throwable e
(:test (:errors (ex-data e))))))]
(vec (repeatedly n attempt-once)))))
;; a couple of quick "attempts" shouldn't trigger the throttler
(expect [:success :success]
(attempt 2))
;; nor should 3
(expect [:success :success :success]
(attempt 3))
;; 4 in quick succession should trigger it
(expect [:success :success :success "Too many attempts! You must wait 0 seconds before trying again."] ; rounded down
(attempt 4))
;; Check that throttling correctly lets you try again after certain delay
(expect [[:success :success :success "Too many attempts! You must wait 0 seconds before trying again."]
[:success]]
[(attempt 4 :a)
(do
(Thread/sleep 6)
(attempt 1 :a))])
;; Next attempt should be throttled, however
(expect [:success "Too many attempts! You must wait 0 seconds before trying again."]
(do
(attempt 4 :b)
(Thread/sleep 6)
(attempt 2 :b)))
;; Sleeping 5+ ms after that shouldn't work due to exponential growth
(expect ["Too many attempts! You must wait 0 seconds before trying again."]
(do
(attempt 4 :c)
(Thread/sleep 6)
(attempt 2 :c)
(Thread/sleep 6)
(attempt 1 :c)))
;; Sleeping 20+ ms however should work
(expect [:success]
(do
(attempt 4 :d)
(Thread/sleep 6)
(attempt 2 :d)
(Thread/sleep 21)
(attempt 1 :d)))
;; Check that the interal list for the throttler doesn't keep growing after throttling starts
(expect [0 3]
[(do (reset! (:attempts test-throttler) '()) ; reset it to 0
(count @(:attempts test-throttler)))
(do (attempt 5)
(count @(:attempts test-throttler)))])
;; Check that attempts clear after the TTL
(expect [0 3 1]
[(do (reset! (:attempts test-throttler) '()) ; reset it to 0
(count @(:attempts test-throttler)))
(do (attempt 3)
(count @(:attempts test-throttler)))
(do (Thread/sleep 25)
(attempt 1)
(count @(:attempts test-throttler)))])
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment