Merge branch 'space_efficient_qb' of github.com:metabase/metabase-init into space_efficient_qb

.gitignore

@@ -19,6 +19,8 @@ profiles.clj
 /*.lock.db
 /*.trace.db
 /resources/frontend_client/app/dist/
+/resources/frontend_client/index.html
 /node_modules/
 /.babel_cache
 /coverage
+/deploy/artifacts/*

circle.yml

@@ -4,6 +4,11 @@ machine:
   java:
     version:
       oraclejdk8
+  python:
+    version: 2.7.3
+dependencies:
+  pre:
+    - pip install awscli==1.7.3
 test:
   override:
     # 0) runs unit tests w/ H2 local DB. Runs against both Mongo + H2 test datasets
@@ -14,3 +19,8 @@ test:
     # 5) runs lein uberjar
     - case $CIRCLE_NODE_INDEX in 0) MB_TEST_DATASETS=h2,mongo,postgres lein test ;; 1) MB_DB_TYPE=postgres MB_DB_DBNAME=circle_test MB_DB_PORT=5432 MB_DB_USER=ubuntu MB_DB_HOST=localhost lein test ;; 2) lein eastwood ;; 3) lein bikeshed --max-line-length 240 ;; 4) npm run lint && npm run build && npm run test ;; 5) CI_DISABLE_WEBPACK_MINIFICATION=1 lein uberjar ;; esac:
         parallel: true
+deployment:
+  master:
+    branch: master
+    commands:
+      - ./deploy/deploy_aws.sh $STACK_ID $APP_ID

deploy/deploy_aws.sh

+#!/bin/bash
+# for reference
+# CIRCLE_SHA1=a3262e9b60a25e6a8a7faa29478b2b455b5ec4a3
+# CIRCLE_BRANCH=master
+
+if [ $# -ne 2 ]; then
+    echo "usage: $0 stackid appid"
+    exit 1
+fi
+
+STACKID=$1
+APPID=$2
+echo "deploying $CIRCLE_SHA1 from $CIRCLE_BRANCH ..."
+aws opsworks create-deployment --stack-id $STACKID --app-id $APPID --comment "deploying $CIRCLE_SHA1 from $CIRCLE_BRANCH" --command='{"Name": "deploy"}'

deploy/deploy_prototype.sh

 #!/bin/bash
+set -eo pipefail
 
 BASEDIR=$(dirname $0)
+source "$BASEDIR/functions"
 
 if [ -z $1 ]; then
   echo "Oops!  You need to specify the name of the EB app version to deploy."
   exit 1
 fi
 
-APP_BUNDLE=$1
-ENVIRONMENT=metabase-proto
-
+EB_VERSION_LABEL=$1
+EB_ENVIRONMENT=metabase-proto
 
 # deploy EB version to environment
-${BASEDIR}/deploy_version.sh ${APP_BUNDLE} ${ENVIRONMENT}
+deploy_version ${EB_ENVIRONMENT} ${EB_VERSION_LABEL}

deploy/deploy_staging.sh

 #!/bin/bash
+set -eo pipefail
 
 BASEDIR=$(dirname $0)
+source "$BASEDIR/functions"
 
-if [ -z $1 ]; then
-  echo "Oops!  You need to specify the name of the EB app version to deploy."
-  exit 1
-fi
-
-APP_BUNDLE=$1
-ENVIRONMENT=metabase-staging
-
+EB_ENVIRONMENT=metabase-staging
 
 # deploy EB version to environment
-${BASEDIR}/deploy_version.sh ${APP_BUNDLE} ${ENVIRONMENT}
+deploy_version ${EB_ENVIRONMENT}

deploy/deploy_version.sh

 #!/bin/bash
+set -eo pipefail
 
 BASEDIR=$(dirname $0)
-
-if [ -z $1 ]; then
-  echo "usage: deploy_version.sh <version> <environment>"
-  exit 1
-fi
-
-if [ -z $2 ]; then
-  echo "usage: deploy_version.sh <version> <environment>"
-  exit 1
-fi
-
-APP_BUNDLE=$1
-ENVIRONMENT=$2
-
-
-# upload app version to EB
-# TODO: check if version already exists
-${BASEDIR}/upload_version.sh ${APP_BUNDLE}
+source "$BASEDIR/functions"
 
 # deploy EB version to environment
-aws elasticbeanstalk update-environment --region us-east-1 --environment-name ${ENVIRONMENT} --version-label ${APP_BUNDLE}
+deploy_version "$1" "$2"

deploy/functions

+#!/bin/bash
+set -eo pipefail
+
+BASEDIR=$(dirname $0)
+PROJECT_ROOT=$(cd ${BASEDIR}/..; pwd)
+
+ARTIFACTS_DIR="$PROJECT_ROOT/deploy/artifacts"
+ARTIFACTS_S3BUCKET=${S3BUCKET:=metabase-artifacts}
+
+BRANCH=$(cd ${PROJECT_ROOT}; $(which git) rev-parse --abbrev-ref HEAD)
+# OpsWorks creates a deploy branch. We'll use master in this case
+[[ "$BRANCH" == "deploy" ]] && BRANCH="master"
+
+COMMITISH=$(cd ${PROJECT_ROOT}; $(which git) rev-parse --short HEAD)
+DATE=$(date +%Y-%m-%d)
+DEFAULT_RELEASE_ZIP_FILE_NAME="metabase-$BRANCH-$DATE-$COMMITISH.zip"
+
+build_uberjar() {
+    echo "building uberjar"
+    lein uberjar
+}
+
+upload_release_artifacts() {
+    echo "uploading $ARTIFACTS_DIR/*.jar -> $ARTIFACTS_S3BUCKET/jar/"
+    aws s3 cp $ARTIFACTS_DIR/ s3://$ARTIFACTS_S3BUCKET/jar/ --recursive --exclude "*" --include "*.jar"
+    echo "uploading $ARTIFACTS_DIR/*.zip -> $ARTIFACTS_S3BUCKET/eb/"
+    aws s3 cp $ARTIFACTS_DIR/ s3://$ARTIFACTS_S3BUCKET/eb/ --recursive --exclude "*" --include "*.zip"
+}
+
+mk_release_artifacts() {
+    METABASE_JAR_NAME="metabase-standalone.jar"
+    RELEASE_TYPE="aws-eb-docker"
+    RELEASE_JAR_FILE_NAME=${METABASE_JAR_NAME%-standalone.jar}-$BRANCH-$DATE-$COMMITISH.jar
+    RELEASE_ZIP_FILE_NAME="$1"
+    UBERJAR_DIR="${PROJECT_ROOT}/target/uberjar"
+
+    if [[ -z $RELEASE_ZIP_FILE_NAME ]]; then
+        RELEASE_ZIP_FILE_NAME=$DEFAULT_RELEASE_ZIP_FILE_NAME
+        echo "release name not provided defaulting to $RELEASE_ZIP_FILE_NAME"
+    fi
+
+    RELEASE_FILES="${PROJECT_ROOT}/deploy/${RELEASE_TYPE}"
+    RELEASE_FILE="${PROJECT_ROOT}/${RELEASE_ZIP_FILE_NAME}"
+
+    # package up the release files
+    (cd $RELEASE_FILES; zip -r $RELEASE_FILE * .ebextensions)
+
+    # add the built uberjar
+    (cd $UBERJAR_DIR; cp metabase-*-SNAPSHOT-*.jar $METABASE_JAR_NAME ; zip $RELEASE_FILE $METABASE_JAR_NAME)
+
+    mkdir -p $ARTIFACTS_DIR
+    rm -f $ARTIFACTS_DIR/*
+    mv -f $RELEASE_FILE $ARTIFACTS_DIR/
+    mv -f $UBERJAR_DIR/$METABASE_JAR_NAME $ARTIFACTS_DIR/$RELEASE_JAR_FILE_NAME
+
+    upload_release_artifacts
+}
+
+create_eb_version() {
+    EB_APPLICATION=Metabase
+    EB_VERSION_LABEL=$1
+    S3_KEY=$2
+
+    [[ -z "$EB_VERSION_LABEL" ]] && EB_VERSION_LABEL="$BRANCH-$DATE-$COMMITISH"
+    [[ -z "$S3_KEY" ]] && S3_KEY=$DEFAULT_RELEASE_ZIP_FILE_NAME
+
+    echo "Creating app version in EB"
+    aws elasticbeanstalk create-application-version --no-auto-create-application --region us-east-1 --application-name ${EB_APPLICATION} --version-label ${EB_VERSION_LABEL} --source-bundle S3Bucket="${ARTIFACTS_S3BUCKET}",S3Key="eb/${S3_KEY}"
+}
+
+deploy_version() {
+    EB_ENVIRONMENT=$1
+    EB_VERSION_LABEL=$2
+
+    [[ -z "$EB_ENVIRONMENT" ]] && EB_VERSION_LABEL="metabase-staging" && echo ""
+    [[ -z "$EB_VERSION_LABEL" ]] && EB_VERSION_LABEL="$BRANCH-$DATE-$COMMITISH"
+    aws elasticbeanstalk update-environment --region us-east-1 --environment-name ${EB_ENVIRONMENT} --version-label ${EB_VERSION_LABEL}
+}

deploy/mk_release.sh

 #!/bin/bash
+set -eo pipefail
 
 BASEDIR=$(dirname $0)
+source "$BASEDIR/functions"
 
-PROJECT_ROOT=`cd ${BASEDIR}/..; pwd`
-UBERJAR_DIR="${PROJECT_ROOT}/target/uberjar"
-
-RELEASE_TYPE="aws-eb-docker"
-if [ ! -z $2 ]; then
-  echo $2
-fi
-
-if [ -z $1 ]; then
-  echo "Oops!  You need to specify a name for the release as an argument."
-  exit 1
-fi
-
-RELEASE_FILES="${PROJECT_ROOT}/deploy/${RELEASE_TYPE}"
-RELEASE_FILE="${PROJECT_ROOT}/${1}.zip"
-
-
-# package up the release files
-(cd $RELEASE_FILES; zip -r $RELEASE_FILE * .ebextensions)
-
-# add the built uberjar
-(cd $UBERJAR_DIR; cp metabase-*-SNAPSHOT-*.jar metabase-standalone.jar; zip $RELEASE_FILE metabase-standalone.jar)
-
+build_uberjar
+mk_release_artifacts "$1"

deploy/upload_version.sh

 #!/bin/bash
+set -eo pipefail
 
-if [ -z $1 ]; then
-  echo "Oops!  You need to specify the name of the EB zip file to upload."
-  exit 1
-fi
-
-APP_BUNDLE=$1
-
-UUID=$(cat /dev/urandom | env LC_CTYPE=C tr -dc 'a-zA-Z0-9' | fold -w 16 | head -n 1)
-S3_KEY="${UUID}_${APP_BUNDLE}.zip"
-
-S3_BUCKET=elasticbeanstalk-us-east-1-867555200881
-APPLICATION=Metabase
-
-
-# upload bundle to s3
-echo "Uploading app version to S3"
-aws s3api put-object --bucket ${S3_BUCKET} --key ${S3_KEY} --body ${APP_BUNDLE}.zip
-
-# create EB version
-echo "Creating app version in EB"
-aws elasticbeanstalk create-application-version --no-auto-create-application --region us-east-1 --application-name ${APPLICATION} --version-label ${APP_BUNDLE} --source-bundle S3Bucket="${S3_BUCKET}",S3Key="${S3_KEY}"
+BASEDIR=$(dirname $0)
+source "$BASEDIR/functions"
 
+create_eb_version "$1" "$2"

package.json

@@ -54,6 +54,7 @@
     "eslint-plugin-react": "^2.5.0",
     "extract-text-webpack-plugin": "^0.8.1",
     "glob": "^5.0.10",
+    "html-webpack-plugin": "git://github.com/tlrobinson/html-webpack-plugin.git#562acca0363224f156c0bcc87d064a1e2f72611c",
     "istanbul-instrumenter-loader": "^0.1.3",
     "jasmine-core": "^2.3.4",
     "karma": "^0.12.36",

project.clj

@@ -80,7 +80,8 @@
                               "-Xmx2048m"                             ; hard limit of 2GB so we stop hitting the 4GB container limit on CircleCI
                               "-XX:+CMSClassUnloadingEnabled"         ; let Clojure's dynamically generated temporary classes be GC'ed from PermGen
                               "-XX:+UseConcMarkSweepGC"]}             ; Concurrent Mark Sweep GC needs to be used for Class Unloading (above)
-             :expectations {:injections [(require 'metabase.test-setup)]
+             :expectations {:global-vars {*warn-on-reflection* false}
+                            :injections [(require 'metabase.test-setup)]
                             :resource-paths ["test_resources"]
                             :env {:mb-test-setting-1 "ABCDEFG"}
                             :jvm-opts ["-Dmb.db.in.memory=true"

resources/frontend_client/index.html → resources/frontend_client/index_template.html

@@ -8,9 +8,6 @@
         <meta name="apple-mobile-web-app-status-bar-style" content="black-translucent" />
         <title>Metabase</title>
 
-        <link rel="stylesheet" href="/app/dist/styles.bundle.css"/>
-        <script charset="utf-8" src="/app/dist/vendor.bundle.js"></script>
-        <script charset="utf-8" src="/app/dist/app.bundle.js"></script>
     </head>
 
     <body ng-controller="Corvus">

src/metabase/api/common/throttle.clj

+(ns metabase.api.common.throttle
+  (:require [clojure.math.numeric-tower :as math])
+  (:import (clojure.lang Atom
+                         Keyword)))
+
+;;; # THROTTLING
+;;
+;; A `Throttler` is a simple object used for throttling API endpoints. It keeps track of all calls to an API endpoint
+;; with some value over some past period of time. If the number of calls with this value exceeds some threshold,
+;; an exception is thrown, telling a user they must wait some period of time before trying again.
+;;
+;; ### EXAMPLE
+;;
+;; Let's consider the email throttling done by POST /api/session.
+;; The basic concept here is to keep a list of failed logins over the last hour. This list looks like:
+;;
+;; (["cam@metabase.com" 1438045261132]
+;;  ["cam@metabase.com" 1438045260450]
+;;  ["cam@metabase.com" 1438045259037]
+;;  ["cam@metabase.com" 1438045258204])
+;;
+;; Every time there's a login attempt, push a new pair of [email timestamp (milliseconds)] to the front of the list.
+;; The list is thus automatically ordered by date, and we can drop the portion of the list with logins that are over
+;; an hour old as needed.
+;;
+;; Once a User has reached some number of login attempts over the past hour (e.g. 5), calculate some delay before
+;; they're allowed to try to log in again (e.g., 15 seconds). This number will increase exponentially as the number of
+;; recent failures increases (e.g., 40 seconds for 6 failed attempts, 90 for 7 failed attempts, etc).
+;;
+;; If applicable, calucate the time since the last failed attempt, and throw an exception telling the user the number
+;; of seconds they must wait before trying again.
+;;
+;; ### USAGE
+;;
+;; Define a new throttler with `make-throttler`, overriding default settings as needed.
+;;
+;;    (require '[metabase.api.common.throttle :as throttle])
+;;    (def email-throttler (throttle/make-throttler :email, :attempts-threshold 10))
+;;
+;; Then call `check` within the body of an endpoint with some value to apply throttling.
+;;
+;;    (defendpoint POST [:as {{:keys [email]} :body}]
+;;      (throttle/check email-throttler email)
+;;      ...)
+
+
+;;; # PUBLIC INTERFACE
+
+(declare calculate-delay
+         remove-old-attempts)
+
+(defrecord Throttler [;; Name of the API field/value being checked. Used to generate appropriate API error messages, so
+                      ;; they'll be displayed on the right part of the screen
+                      ^Keyword exception-field-key
+                      ;; [Internal] List of attempt entries. These are pairs of [key timestamp (ms)],
+                      ;; e.g. ["cam@metabase.com" 1438045261132]
+                      ^Atom    attempts
+                      ;; Amount of time to keep an entry in ATTEMPTS before dropping it.
+                      ^Integer attempt-ttl-ms
+                      ;; Number of attempts allowed with a given key before throttling is applied.
+                      ^Integer attempts-threshold
+                      ;; Once throttling is in effect, initial delay before allowing another attempt. This grows
+                      ;; according to DELAY-EXPONENT.
+                      ^Integer initial-delay-ms
+                      ;; For each subsequent failure past ATTEMPTS-THRESHOLD, increase the delay to
+                      ;; INITIAL-DELAY-MS * (num-attempts-over-theshold ^ DELAY-EXPONENT). e.g. if INITIAL-DELAY-MS is 15
+                      ;; and DELAY-EXPONENT is 2, the first attempt past ATTEMPTS-THRESHOLD will require the user to wait
+                      ;; 15 seconds (15 * 1^2), the next attempt after that 60 seconds (15 * 2^2), then 135, and so on.
+                      ^Integer delay-exponent])
+
+;; These are made private because you should use `make-throttler` instead.
+(alter-meta! #'->Throttler assoc :private true)
+(alter-meta! #'map->Throttler assoc :private true)
+
+(def ^:private ^:const throttler-defaults
+  {:initial-delay-ms   (* 15 1000)
+   :attempts-threshold 10
+   :delay-exponent     1.5
+   :attempt-ttl-ms     (* 1000 60 60)})
+
+(defn make-throttler
+  "Create a new `Throttler`.
+
+     (require '[metabase.api.common.throttle :as throttle])
+     (def email-throttler (throttle/make-throttler :email, :attempts-threshold 10))"
+  [exception-field-key & {:as kwargs}]
+  (map->Throttler (merge throttler-defaults kwargs {:attempts            (atom '())
+                                                    :exception-field-key exception-field-key})))
+
+(defn check
+  "Throttle an API call based on values of KEYY. Each call to this function will record KEYY to THROTTLER's internal list;
+   if the number of entires containing KEYY exceed THROTTLER's thresholds, throw an exception.
+
+     (defendpoint POST [:as {{:keys [email]} :body}]
+       (throttle/check email-throttler email)
+       ...)"
+  [^Throttler {:keys [attempts exception-field-key], :as throttler} keyy] ; technically, keyy can be nil so you can record *all* attempts
+  {:pre [(= (type throttler) Throttler)]}
+  (remove-old-attempts throttler)
+  (when-let [delay-ms (calculate-delay throttler keyy)]
+    (let [message (format "Too many attempts! You must wait %d seconds before trying again."
+                          (int (math/round (/ delay-ms 1000))))]
+      (throw (ex-info message {:status-code 400
+                               :errors      {exception-field-key message}}))))
+  (swap! attempts conj [keyy (System/currentTimeMillis)]))
+
+
+;;; # INTERNAL IMPLEMENTATION
+
+(defn- remove-old-attempts
+  "Remove THROTTLER entires past the TTL."
+  [^Throttler {:keys [attempts attempt-ttl-ms]}]
+  (let [old-attempt-cutoff (- (System/currentTimeMillis) attempt-ttl-ms)
+        non-old-attempt?   (fn [[_ timestamp]]
+                             (> timestamp old-attempt-cutoff))]
+    (reset! attempts (take-while non-old-attempt? @attempts))))
+
+(defn- calculate-delay
+  "Calculate the delay in milliseconds, if any, that should be applied to a given THROTTLER / KEYY combination."
+  ([^Throttler throttler keyy]
+   (calculate-delay throttler keyy (System/currentTimeMillis)))
+
+  ([^Throttler {:keys [attempts initial-delay-ms attempts-threshold delay-exponent]} keyy current-time-ms]
+   (let [[[_ most-recent-attempt-ms], :as keyy-attempts] (filter (fn [[k _]] (= k keyy)) @attempts)]
+     (when most-recent-attempt-ms
+       (let [num-recent-attempts         (count keyy-attempts)
+             num-attempts-over-threshold (- (inc num-recent-attempts) attempts-threshold)] ; add one to the sum to account for the current attempt
+         (when (> num-attempts-over-threshold 0)
+           (let [delay-ms              (* (math/expt num-attempts-over-threshold delay-exponent)
+                                          initial-delay-ms)
+                 next-login-allowed-at (+ most-recent-attempt-ms delay-ms)
+                 ms-till-next-login    (- next-login-allowed-at current-time-ms)]
+             (when (> ms-till-next-login 0)
+               ms-till-next-login))))))))

src/metabase/api/session.clj

@@ -6,6 +6,7 @@
             [hiccup.core :refer [html]]
             [korma.core :as k]
             [metabase.api.common :refer :all]
+            [metabase.api.common.throttle :as throttle]
             [metabase.db :refer :all]
             [metabase.email.messages :as email]
             (metabase.models [user :refer [User set-user-password set-user-password-reset-token]]
@@ -24,17 +25,25 @@
     session-id))
 
 
+;;; ## API Endpoints
+
+(def ^:private login-throttlers
+  {:email      (throttle/make-throttler :email)
+   :ip-address (throttle/make-throttler :email, :attempts-threshold 50)}) ; IP Address doesn't have an actual UI field so just show error by email
+
 (defendpoint POST "/"
   "Login."
-  [:as {{:keys [email password] :as body} :body}]
+  [:as {{:keys [email password] :as body} :body, remote-address :remote-addr}]
   {email    [Required Email]
    password [Required NonEmptyString]}
-  (let [user (sel :one :fields [User :id :password_salt :password] :email email (k/where {:is_active true}))]
-    (checkp (not (nil? user))
-    ; Don't leak whether the account doesn't exist or the password was incorrect
-      'password "did not match stored password")
-    (checkp (pass/verify-password password (:password_salt user) (:password user))
-      'password "did not match stored password")
+  (throttle/check (login-throttlers :ip-address) remote-address)
+  (throttle/check (login-throttlers :email)      email)
+  (let [user (sel :one :fields [User :id :password_salt :password], :email email (k/where {:is_active true}))]
+    ;; Don't leak whether the account doesn't exist or the password was incorrect
+    (when-not (and user
+                   (pass/verify-password password (:password_salt user) (:password user)))
+      (throw (ex-info "Password did not match stored password." {:status-code 400
+                                                                 :errors      {:password "did not match stored password"}})))
     (let [session-id (create-session (:id user))]
       {:id session-id})))
 
@@ -53,10 +62,16 @@
 ;;
 ;; There's also no need to salt the token because it's already random <3
 
+(def ^:private forgot-password-throttlers
+  {:email      (throttle/make-throttler :email)
+   :ip-address (throttle/make-throttler :email, :attempts-threshold 50)})
+
 (defendpoint POST "/forgot_password"
   "Send a reset email when user has forgotten their password."
-  [:as {:keys [server-name] {:keys [email]} :body, :as request}]
+  [:as {:keys [server-name] {:keys [email]} :body, remote-address :remote-addr, :as request}]
   {email [Required Email]}
+  (throttle/check (forgot-password-throttlers :ip-address) remote-address)
+  (throttle/check (forgot-password-throttlers :email)      email)
   ;; Don't leak whether the account doesn't exist, just pretend everything is ok
   (when-let [user-id (sel :one :id User :email email)]
     (let [reset-token        (set-user-password-reset-token user-id)

src/metabase/db.clj

@@ -95,7 +95,6 @@
   [& {:keys [auto-migrate]
       :or {auto-migrate true}}]
   (reset! setup-db-has-been-called? true)
-  (log/info "Setting up DB specs...")
 
   ;; Test DB connection and throw exception if we have any troubles connecting
   (log/info "Verifying Database Connection ...")

src/metabase/driver.clj

@@ -125,7 +125,7 @@
   (let [-sync-database! (u/runtime-resolved-fn 'metabase.driver.sync 'sync-database!)] ; these need to be resolved at runtime to avoid circular deps
     (fn [database]
       {:pre [(map? database)]}
-      (time (-sync-database! (engine->driver (:engine database)) database)))))
+      (-sync-database! (engine->driver (:engine database)) database))))
 
 (def ^{:arglists '([table])} sync-table!
   "Sync a `Table` and its `Fields`."
@@ -183,10 +183,8 @@
           (when (= :failed (:status query-result))
             (throw (Exception. ^String (get query-result :error "general error"))))
           (query-complete query-execution query-result))
-        (catch Exception ex
-          (log/warn ex)
-          (.printStackTrace ex)
-          (query-fail query-execution (.getMessage ex)))))))
+        (catch Exception e
+          (query-fail query-execution (.getMessage e)))))))
 
 (defn query-fail
   "Save QueryExecution state and construct a failed query response"

src/metabase/driver/query_processor.clj

@@ -41,7 +41,6 @@
   (fn [query]
     (try (qp query)
          (catch Throwable e
-           (.printStackTrace e)
            {:status     :failed
             :error      (.getMessage e)
             :stacktrace (u/filtered-stacktrace e)

src/metabase/driver/sync.clj

@@ -43,39 +43,41 @@
             *sel-disable-logging* true]
     (sync-in-context driver database
       (fn []
-        (log/info (u/format-color 'magenta "Syncing %s database '%s'..." (name (:engine database)) (:name database)))
-
-        (let [active-table-names (active-table-names driver database)
-              table-name->id     (sel :many :field->id [Table :name] :db_id (:id database) :active true)]
-          (assert (set? active-table-names) "active-table-names should return a set.")
-          (assert (every? string? active-table-names) "active-table-names should return the names of Tables as *strings*.")
-
-          ;; First, let's mark any Tables that are no longer active as such.
-          ;; These are ones that exist in table-name->id but not in active-table-names.
-          (doseq [[table-name table-id] table-name->id]
-            (when-not (contains? active-table-names table-name)
-              (upd Table table-id :active false)
-              (log/info (u/format-color 'cyan "Marked table %s.%s as inactive." (:name database) table-name))
-
-              ;; We need to mark driver Table's Fields as inactive so we don't expose them in UI such as FK selector (etc.)
-              (k/update Field
-                        (k/where {:table_id table-id})
-                        (k/set-fields {:active false}))))
-
-          ;; Next, we'll create new Tables (ones that came back in active-table-names but *not* in table-name->id)
-          (let [existing-table-names (set (keys table-name->id))
-                new-table-names      (set/difference active-table-names existing-table-names)]
-            (when (seq new-table-names)
-              (log/info (u/format-color 'blue "Found new tables: %s" new-table-names))
-              (doseq [new-table-name new-table-names]
-                (ins Table :db_id (:id database), :active true, :name new-table-name)))))
-
-        ;; Now sync the active tables
-        (->> (sel :many Table :db_id (:id database) :active true)
-             (map #(assoc % :db (delay database))) ; replace default delays with ones that reuse database (and don't require a DB call)
-             (sync-database-active-tables! driver))
-
-        (log/info (u/format-color 'magenta "Finished syncing %s database %s." (name (:engine database)) (:name database)))))))
+        (let [start-time (System/currentTimeMillis)]
+          (log/info (u/format-color 'magenta "Syncing %s database '%s'..." (name (:engine database)) (:name database)))
+
+          (let [active-table-names (active-table-names driver database)
+                table-name->id     (sel :many :field->id [Table :name] :db_id (:id database) :active true)]
+            (assert (set? active-table-names) "active-table-names should return a set.")
+            (assert (every? string? active-table-names) "active-table-names should return the names of Tables as *strings*.")
+
+            ;; First, let's mark any Tables that are no longer active as such.
+            ;; These are ones that exist in table-name->id but not in active-table-names.
+            (doseq [[table-name table-id] table-name->id]
+              (when-not (contains? active-table-names table-name)
+                (upd Table table-id :active false)
+                (log/info (u/format-color 'cyan "Marked table %s.%s as inactive." (:name database) table-name))
+
+                ;; We need to mark driver Table's Fields as inactive so we don't expose them in UI such as FK selector (etc.)
+                (k/update Field
+                          (k/where {:table_id table-id})
+                          (k/set-fields {:active false}))))
+
+            ;; Next, we'll create new Tables (ones that came back in active-table-names but *not* in table-name->id)
+            (let [existing-table-names (set (keys table-name->id))
+                  new-table-names      (set/difference active-table-names existing-table-names)]
+              (when (seq new-table-names)
+                (log/debug (u/format-color 'blue "Found new tables: %s" new-table-names))
+                (doseq [new-table-name new-table-names]
+                  (ins Table :db_id (:id database), :active true, :name new-table-name)))))
+
+          ;; Now sync the active tables
+          (->> (sel :many Table :db_id (:id database) :active true)
+               (map #(assoc % :db (delay database))) ; replace default delays with ones that reuse database (and don't require a DB call)
+               (sync-database-active-tables! driver))
+
+          (log/info (u/format-color 'magenta "Finished syncing %s database %s. (%d ms)" (name (:engine database)) (:name database)
+                                    (- (System/currentTimeMillis) start-time))))))))
 
 (defn sync-table!
   "Sync a *single* TABLE by running all the sync steps for it.
@@ -146,7 +148,7 @@
 
         (sync-table-fields-metadata! driver table)
         (swap! finished-tables-count inc)
-        (log/info (u/format-color 'magenta "%s Synced table '%s'." (sync-progress-meter-string @finished-tables-count tables-count) (:name table)))))))
+        (log/debug (u/format-color 'magenta "%s Synced table '%s'." (sync-progress-meter-string @finished-tables-count tables-count) (:name table)))))))
 
 
 ;; ## sync-table steps.
@@ -186,7 +188,7 @@
   {:pre [(set? pk-fields)
          (every? string? pk-fields)]}
   (doseq [{field-name :name field-id :id} (sel :many :fields [Field :name :id], :table_id (:id table), :special_type nil, :name [in pk-fields], :parent_id nil)]
-    (log/info (u/format-color 'green "Field '%s.%s' is a primary key. Marking it as such." (:name table) field-name))
+    (log/debug (u/format-color 'green "Field '%s.%s' is a primary key. Marking it as such." (:name table) field-name))
     (upd Field field-id :special_type :id)))
 
 (defn- sync-table-active-fields-and-pks!
@@ -212,7 +214,7 @@
       (let [existing-field-names (set (keys existing-field-name->field))
             new-field-names      (set/difference (set (keys active-column-names->type)) existing-field-names)]
         (when (seq new-field-names)
-          (log/info (u/format-color 'blue "Found new fields for table '%s': %s" (:name table) new-field-names)))
+          (log/debug (u/format-color 'blue "Found new fields for table '%s': %s" (:name table) new-field-names)))
         (doseq [[active-field-name active-field-type] active-column-names->type]
           ;; If Field doesn't exist create it
           (if-not (contains? existing-field-names active-field-name)
@@ -223,7 +225,7 @@
             ;; Otherwise update the Field type if needed
             (let [{existing-base-type :base_type, existing-field-id :id} (existing-field-name->field active-field-name)]
               (when-not (= active-field-type existing-base-type)
-                (log/info (u/format-color 'blue "Field '%s.%s' has changed from a %s to a %s." (:name table) active-field-name existing-base-type active-field-type))
+                (log/debug (u/format-color 'blue "Field '%s.%s' has changed from a %s to a %s." (:name table) active-field-name existing-base-type active-field-type))
                 (upd Field existing-field-id :base_type active-field-type))))))
       ;; TODO - we need to add functionality to update nested Field base types as well!
 
@@ -262,7 +264,7 @@
             (when-let [fk-column-id (fk-name->id fk-column-name)]
               (when-let [dest-table-id (table-name->id dest-table-name)]
                 (when-let [dest-column-id (sel :one :id Field, :table_id dest-table-id, :name dest-column-name, :parent_id nil)]
-                  (log/info (u/format-color 'green "Marking foreign key '%s.%s' -> '%s.%s'." (:name table) fk-column-name dest-table-name dest-column-name))
+                  (log/debug (u/format-color 'green "Marking foreign key '%s.%s' -> '%s.%s'." (:name table) fk-column-name dest-table-name dest-column-name))
                   (ins ForeignKey
                     :origin_id      fk-column-id
                     :destination_id dest-column-id
@@ -329,7 +331,7 @@
   [field]
   (when (nil? (:display_name field))
     (let [display-name (common/name->human-readable-name (:name field))]
-      (log/info (u/format-color 'green "Field '%s.%s' has no display_name. Setting it now." (:name @(:table field)) (:name field) display-name))
+      (log/debug (u/format-color 'green "Field '%s.%s' has no display_name. Setting it now." (:name @(:table field)) (:name field) display-name))
       (upd Field (:id field) :display_name display-name)
       (assoc field :display_name display-name))))
 
@@ -371,7 +373,7 @@
       (assert (>= percent-urls 0.0))
       (assert (<= percent-urls 100.0))
       (when (> percent-urls percent-valid-url-threshold)
-        (log/info (u/format-color 'green "Field '%s' is %d%% URLs. Marking it as a URL." @(:qualified-name field) (int (math/round (* 100 percent-urls)))))
+        (log/debug (u/format-color 'green "Field '%s' is %d%% URLs. Marking it as a URL." @(:qualified-name field) (int (math/round (* 100 percent-urls)))))
         (upd Field (:id field) :special_type :url)
         (assoc field :special_type :url)))))
 
@@ -388,7 +390,7 @@
   (let [cardinality (queries/field-distinct-count field low-cardinality-threshold)]
     (when (and (> cardinality 0)
                (< cardinality low-cardinality-threshold))
-      (log/info (u/format-color 'green "Field '%s' has %d unique values. Marking it as a category." @(:qualified-name field) cardinality))
+      (log/debug (u/format-color 'green "Field '%s' has %d unique values. Marking it as a category." @(:qualified-name field) cardinality))
       (upd Field (:id field) :special_type :category)
       (assoc field :special_type :category))))
 
@@ -431,7 +433,7 @@
     (let [avg-len (field-avg-length driver field)]
       (assert (integer? avg-len) "field-avg-length should return an integer.")
       (when (> avg-len average-length-no-preview-threshold)
-        (log/info (u/format-color 'green "Field '%s' has an average length of %d. Not displaying it in previews." @(:qualified-name field) avg-len))
+        (log/debug (u/format-color 'green "Field '%s' has an average length of %d. Not displaying it in previews." @(:qualified-name field) avg-len))
         (upd Field (:id field) :preview_display false)
         (assoc field :preview_display false)))))
 
@@ -465,7 +467,7 @@
              (contains? #{:CharField :TextField} (:base_type field))
              (values-are-valid-json? (->> (field-values-lazy-seq driver field)
                                           (take max-sync-lazy-seq-results))))
-    (log/info (u/format-color 'green "Field '%s' looks like it contains valid JSON objects. Setting special_type to :json." @(:qualified-name field)))
+    (log/debug (u/format-color 'green "Field '%s' looks like it contains valid JSON objects. Setting special_type to :json." @(:qualified-name field)))
     (upd Field (:id field) :special_type :json, :preview_display false)
     (assoc field :special_type :json, :preview_display false)))
 
@@ -541,7 +543,7 @@
   [field]
   (when-not (:special_type field)
     (when-let [[pattern _ special-type] (field->name-inferred-special-type field)]
-      (log/info (u/format-color 'green "%s '%s' matches '%s'. Setting special_type to '%s'."
+      (log/debug (u/format-color 'green "%s '%s' matches '%s'. Setting special_type to '%s'."
                                 (name (:base_type field)) @(:qualified-name field) pattern (name special-type)))
       (upd Field (:id field) :special_type special-type)
       (assoc field :special_type special-type))))
@@ -564,7 +566,7 @@
         ;; OK, now create new Field objects for ones that came back from active-nested-field-name->type but *aren't* in existing-nested-field-name->id
         (doseq [[nested-field-name nested-field-type] nested-field-name->type]
           (when-not (contains? (set (map keyword (keys existing-nested-field-name->id))) (keyword nested-field-name))
-            (log/info (u/format-color 'blue "Found new nested field: '%s.%s'" @(:qualified-name field) (name nested-field-name)))
+            (log/debug (u/format-color 'blue "Found new nested field: '%s.%s'" @(:qualified-name field) (name nested-field-name)))
             (let [nested-field (ins Field, :table_id (:table_id field), :parent_id (:id field), :name (name nested-field-name) :base_type (name nested-field-type), :active true)]
               ;; Now recursively sync this nested Field
               ;; Replace parent so deref doesn't need to do a DB call

test/metabase/api/common/throttle_test.clj

+(ns metabase.api.common.throttle-test
+  (:require [expectations :refer :all]
+            [metabase.api.common.throttle :as throttle]
+            [metabase.test.util :refer [resolve-private-fns]]))
+
+(def ^:private test-throttler (throttle/make-throttler :test, :initial-delay-ms 2, :attempts-threshold 3, :delay-exponent 2, :attempt-ttl-ms 10))
+
+;;; # tests for calculate-delay
+(resolve-private-fns metabase.api.common.throttle calculate-delay)
+
+;; no delay should be calculated for the 3rd attempt
+(expect nil
+  (do (reset! (:attempts test-throttler) '([:x 100],[:x 99]))
+      (calculate-delay test-throttler :x 101)))
+
+;; 1 ms delay on 4th attempt 1ms after the last
+(expect 1
+  (do (reset! (:attempts test-throttler) '([:x 100], [:x 99], [:x 98]))
+      (calculate-delay test-throttler :x 101)))
+
+;; 2 ms after last attempt, they should be allowed to try again
+(expect nil
+  (do (reset! (:attempts test-throttler) '([:x 100], [:x 99], [:x 98]))
+      (calculate-delay test-throttler :x 102)))
+
+;; However if this was instead the 5th attempt delay should grow exponentially (2 * 2^2 = 8), - 2 ms = 6
+(expect 6
+  (do (reset! (:attempts test-throttler) '([:x 100], [:x 99], [:x 98], [:x 97]))
+      (calculate-delay test-throttler :x 102)))
+
+;; Should be allowed after 6 more secs
+(expect nil
+  (do (reset! (:attempts test-throttler) '([:x 100], [:x 99], [:x 98], [:x 97]))
+      (calculate-delay test-throttler :x 108)))
+
+;; Check that delay keeps growing according to delay-exponent (2 * 3^2 = 2 * 9 = 18)
+(expect 18
+  (do (reset! (:attempts test-throttler) '([:x 108], [:x 100], [:x 99], [:x 98], [:x 97]))
+      (calculate-delay test-throttler :x 108)))
+
+
+;;; # tests for check
+
+(defn- attempt
+  ([n]
+   (attempt n (gensym)))
+  ([n k]
+   (let [attempt-once (fn []
+                      (try
+                        (throttle/check test-throttler k)
+                        :success
+                        (catch Throwable e
+                          (:test (:errors (ex-data e))))))]
+     (vec (repeatedly n attempt-once)))))
+
+;; a couple of quick "attempts" shouldn't trigger the throttler
+(expect [:success :success]
+  (attempt 2))
+
+;; nor should 3
+(expect [:success :success :success]
+  (attempt 3))
+
+;; 4 in quick succession should trigger it
+(expect [:success :success :success "Too many attempts! You must wait 0 seconds before trying again."] ; rounded down
+  (attempt 4))
+
+;; Check that throttling correctly lets you try again after certain delay
+(expect [[:success :success :success "Too many attempts! You must wait 0 seconds before trying again."]
+         [:success]]
+  [(attempt 4 :a)
+   (do
+     (Thread/sleep 2)
+     (attempt 1 :a))])
+
+;; Next attempt should be throttled, however
+(expect [:success "Too many attempts! You must wait 0 seconds before trying again."]
+  (do
+    (attempt 4 :b)
+    (Thread/sleep 2)
+    (attempt 2 :b)))
+
+;; Sleeping 2 ms after that shouldn't work due to exponential growth
+(expect ["Too many attempts! You must wait 0 seconds before trying again."]
+  (do
+    (attempt 4 :c)
+    (Thread/sleep 2)
+    (attempt 2 :c)
+    (Thread/sleep 2)
+    (attempt 1 :c)))
+
+;; Sleeping 8 ms however should work
+(expect [:success]
+  (do
+    (attempt 4 :d)
+    (Thread/sleep 2)
+    (attempt 2 :d)
+    (Thread/sleep 8)
+    (attempt 1 :d)))
+
+;; Check that the interal list for the throttler doesn't keep growing after throttling starts
+(expect [0 3]
+  [(do (reset! (:attempts test-throttler) '()) ; reset it to 0
+       (count @(:attempts test-throttler)))
+   (do (attempt 5)
+       (count @(:attempts test-throttler)))])
+
+;; Check that attempts clear after the TTL
+(expect [0 3 1]
+  [(do (reset! (:attempts test-throttler) '()) ; reset it to 0
+       (count @(:attempts test-throttler)))
+   (do (attempt 3)
+       (count @(:attempts test-throttler)))
+   (do (Thread/sleep 10)
+       (attempt 1)
+       (count @(:attempts test-throttler)))])

test/metabase/api/session_test.clj

@@ -34,6 +34,19 @@
   (client :post 400 "session" (-> (user->credentials :rasta)
                                   (assoc :password "something else"))))
 
+;; Test that people get blocked from attempting to login if they try too many times
+;; (Check that throttling works at the API level -- more tests in metabase.api.common.throttle-test)
+(expect
+    [{:errors {:email "Too many attempts! You must wait 15 seconds before trying again."}}
+     {:errors {:email "Too many attempts! You must wait 15 seconds before trying again."}}]
+  (let [login #(client :post 400 "session" {:email "fakeaccount3000@metabase.com", :password "toucans"})]
+    ;; attempt to log in 10 times
+    (dorun (repeatedly 10 login))
+    ;; throttling should now be triggered
+    [(login)
+     ;; Trying to login immediately again should still return throttling error
+     (login)]))
+
 
 ;; ## DELETE /api/session
 ;; Test that we can logout