Merge branch 'master' into use_annotations

circle.yml

 test:
   override:
-    - case $CIRCLE_NODE_INDEX in 0) lein eastwood ;; 1) lein expectations ;; esac:
+    - case $CIRCLE_NODE_INDEX in 0) lein eastwood ;; 1) lein test ;; esac:
         parallel: true

project.clj

@@ -5,6 +5,7 @@
   :description "Metabase Community Edition"
   :url "http://metabase.com/"
   :min-lein-version "2.3.0"
+  :aliases {"test" ["with-profile" "+expectations" "expectations"]}
   :dependencies [[org.clojure/clojure "1.6.0"]
                  [org.clojure/core.async "LATEST"]                    ; facilities for async programming + communication (using 'LATEST' because this is an alpha library)
                  [org.clojure/core.match "0.3.0-alpha4"]              ; optimized pattern matching library for Clojure
@@ -69,5 +70,8 @@
                               "-XX:MaxPermSize=128m"                  ; a little more headroom for PermGen
                               "-XX:+CMSClassUnloadingEnabled"         ; let Clojure's dynamically generated temporary classes be GC'ed from PermGen
                               "-XX:+UseConcMarkSweepGC"]}             ; Concurrent Mark Sweep GC needs to be used for Class Unloading (above)
+             :expectations {:jvm-opts ["-Dmb.db.file=target/metabase-test"
+                                       "-Dmb.jetty.join=false"
+                                       "-Dmb.jetty.port=3001"]}
              :uberjar {:aot :all
                        :prep-tasks ["npm" "gulp" "javac" "compile"]}})

src/metabase/core.clj

@@ -53,37 +53,57 @@
           hostname (or (config/config-str :mb-jetty-host) "localhost")
           port (config/config-int :mb-jetty-port)
           setup-url (str "http://"
-                      (or hostname "localhost")
-                      (when-not (= 80 port) (str ":" port))
-                      "/setup/init/"
-                      setup-token)]
+                         (or hostname "localhost")
+                         (when-not (= 80 port) (str ":" port))
+                         "/setup/init/"
+                         setup-token)]
       (log/info (str "Please use the following url to setup your Metabase installation:\n\n"
-                  setup-url
-                  "\n\n"))))
+                     setup-url
+                     "\n\n"))))
 
   (log/info "Metabase Initialization COMPLETE")
   true)
 
 
+;; ## Jetty (Web) Server
+
+
+(def ^:private jetty-instance
+  (atom nil))
+
+(defn start-jetty
+  "Start the embedded Jetty web server."
+  []
+  (when-not @jetty-instance
+    (let [jetty-config (cond-> (medley/filter-vals identity {:port (config/config-int :mb-jetty-port)
+                                                             :host (config/config-str :mb-jetty-host)
+                                                             :max-threads (config/config-int :mb-jetty-maxthreads)
+                                                             :min-threads (config/config-int :mb-jetty-minthreads)
+                                                             :max-queued (config/config-int :mb-jetty-maxqueued)
+                                                             :max-idle-time (config/config-int :mb-jetty-maxidletime)})
+                               (config/config-str :mb-jetty-join) (assoc :join? (config/config-bool :mb-jetty-join))
+                               (config/config-str :mb-jetty-daemon) (assoc :daemon? (config/config-bool :mb-jetty-daemon)))]
+      (log/info "Launching Embedded Jetty Webserver with config:\n" (with-out-str (clojure.pprint/pprint jetty-config)))
+      (->> (ring-jetty/run-jetty app jetty-config)
+           (reset! jetty-instance)))))
+
+(defn stop-jetty
+  "Stop the embedded Jetty web server."
+  []
+  (when @jetty-instance
+    (log/info "Shutting Down Embedded Jetty Webserver")
+    (.stop ^org.eclipse.jetty.server.Server @jetty-instance)
+    (reset! jetty-instance nil)))
+
+
 (defn -main
   "Launch Metabase in standalone mode."
   [& args]
   (log/info "Starting Metabase in STANDALONE mode")
-
-  ;; run application init and if there are no issues then continue on to launching the webserver
-  (when (try
-          (init)
-          (catch Exception e
-            (log/error "Metabase Initialization FAILED: " (.getMessage e))))
-    ;; startup webserver
-    (let [jetty-config (->> {:port (config/config-int :mb-jetty-port)
-                             :host (config/config-str :mb-jetty-host)
-                             :join? (config/config-bool :mb-jetty-join?)
-                             :daemon? (config/config-bool :mb-jetty-daemon?)
-                             :max-threads (config/config-int :mb-jetty-maxthreads)
-                             :min-threads (config/config-int :mb-jetty-minthreads)
-                             :max-queued (config/config-int :mb-jetty-maxqueued)
-                             :max-idle-time (config/config-int :mb-jetty-maxidletime)}
-                         (medley/filter-vals identity))]
-      (log/info "Launching Embedded Jetty Webserver with config:\n" (with-out-str (clojure.pprint/pprint jetty-config)))
-      (ring-jetty/run-jetty app jetty-config))))
+  (try
+    ;; run our initialization process
+    (init)
+    ;; launch embedded webserver
+    (start-jetty)
+    (catch Exception e
+      (log/error "Metabase Initialization FAILED: " (.getMessage e)))))

test/log4j.properties

+log4j.rootLogger=INFO, console
+
+# log to the console
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.Target=System.out
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+log4j.appender.console.layout.ConversionPattern=%d [%-5p] %c :: %m%n
+
+# log to a file
+log4j.appender.file=org.apache.log4j.RollingFileAppender
+log4j.appender.file.File=${logfile.path}/corvus.log
+log4j.appender.file.MaxFileSize=500MB
+log4j.appender.file.MaxBackupIndex=2
+log4j.appender.file.layout=org.apache.log4j.PatternLayout
+log4j.appender.file.layout.ConversionPattern=%d [%t] %-5p%c - %m%n
+
+# customizations to logging by package
+log4j.logger.com.mchange=WARN
+log4j.logger.liquibase=WARN
+log4j.logger.metabase=INFO

test/metabase/api/common_test.clj

@@ -5,7 +5,7 @@
             [metabase.api.org-test :refer [create-org]]
             [metabase.test-data :refer :all]
             [metabase.test-data.create :refer [create-user]]
-            metabase.test-utils
+            metabase.test-setup
             [metabase.test.util :refer :all]
             [metabase.util :refer [regex= regex?]])
   (:import com.metabase.corvus.api.ApiException))

test/metabase/api/meta/field_test.clj

@@ -55,6 +55,7 @@
 
 ;; ## PUT /api/meta/field/:id
 ;; Check that we can update a Field
+;; TODO - this should NOT be modifying a field from our test data, we should create new data to mess with
 (expect-eval-actual-first
     (match-$ (let [field (sel :one Field :id (field->id :venues :latitude))]
                ;; this is sketchy. But return the Field back to its unmodified state so it won't affect other unit tests
@@ -73,4 +74,4 @@
        :preview_display true
        :created_at $
        :base_type "FloatField"})
-      ((user->client :rasta) :put 200 (format "meta/field/%d" (field->id :venues :latitude)) {:special_type :fk}))
+  ((user->client :rasta) :put 200 (format "meta/field/%d" (field->id :venues :latitude)) {:special_type :fk}))

test/metabase/http_client.clj

@@ -3,6 +3,7 @@
   (:require [clojure.tools.logging :as log]
             [cheshire.core :as cheshire]
             [clj-http.lite.client :as client]
+            [metabase.config :as config]
             [metabase.util :as u])
   (:import com.metabase.corvus.api.ApiException))
 
@@ -15,7 +16,7 @@
 
 (def ^:dynamic *url-prefix*
   "Prefix to automatically prepend to the URL of calls made with `client`."
-  "http://localhost:3000/api/")
+  (str "http://localhost:" (config/config-str :mb-jetty-port) "/api/"))
 
 (defn
   ^{:arglists ([credentials? method expected-status-code? url http-body-map? & url-kwargs])}

test/metabase/models/org_perm_test.clj

-(ns metabase.models.org-perm-test
-  (:require [clojure.tools.logging :as log]
-            [clj-time.coerce :as tc]
-            [clj-time.core :as time]
-            [expectations :refer :all]
-            [korma.core :refer :all]
-            (metabase [db :refer :all]
-                      test-utils)
-            (metabase.models [org-perm :refer [OrgPerm]]
-                             [org :refer [Org]]
-                             [user :refer [User]])))
-
-
-;(defn insert-org []
-;  (let [result (ins Org
-;                 :name "org_perm_test"
-;                 :slug "org_perm_test"
-;                 :inherits false)]
-;    (or (:id result) -1)))
-;
-;(defn insert-user []
-;  (let [result (ins User
-;                 :email "org_perm_test"
-;                 :first_name "org_perm_test"
-;                 :last_name "org_perm_test"
-;                 :password "org_perm_test"
-;                 :date_joined (java.sql.Timestamp. (tc/to-long (time/now)))
-;                 :is_active true
-;                 :is_staff true
-;                 :is_superuser false)]
-;    (or (:id result) -1)))
-;
-;(defn count-perms []
-;  (get-in (first (select OrgPerm (aggregate (count :*) :cnt))) [:cnt]))
-;
-;
-;; start with 0 entries
-;(expect 0 (count-perms))
-;
-;; insert a new value
-;(expect (more->
-;          false nil?
-;          true (contains? :id))
-;  (let [org-id (insert-org)
-;        user-id (insert-user)]
-;    (ins OrgPerm
-;      :admin false
-;      :organization_id org-id
-;      :user_id user-id)))
-;
-;; now we should have 1 entry
-;(expect 1 (count-perms))

test/metabase/models/org_test.clj

-(ns metabase.models.org-test
-  (:require [clojure.tools.logging :as log]
-            [expectations :refer :all]
-            [korma.core :refer :all]
-            (metabase [db :refer :all]
-                      test-utils)
-            [metabase.models.org :refer [Org]]))
-
-
-;(defn count-orgs []
-;  (get-in (first (select Org (aggregate (count :*) :cnt))) [:cnt]))
-;
-;
-;; start with 0 entries
-;(expect 0 (count-orgs))
-;
-;; insert a new value
-;(expect (more->
-;          false nil?
-;          true (contains? :id))
-;  (ins Org
-;    :name "org_test"
-;    :slug "org_test"
-;    :inherits false))
-;
-;; now we should have 1 entry
-;(expect 1 (count-orgs))

test/metabase/models/setting_test.clj

@@ -2,8 +2,7 @@
   (:require [expectations :refer :all]
             [medley.core :as m]
             (metabase [db :refer [sel]]
-                      [test-data :refer :all]
-                      test-utils)
+                      [test-data :refer :all])
             [metabase.models.setting :refer [defsetting Setting] :as setting]
             [metabase.test.util :refer :all]))
 

test/metabase/models/user_test.clj

-(ns metabase.models.user-test
-  (:require [clojure.tools.logging :as log]
-            [clj-time.coerce :as tc]
-            [clj-time.core :as time]
-            [expectations :refer :all]
-            [korma.core :refer :all]
-            (metabase [db :refer :all]
-                      test-utils)
-            [metabase.models.user :refer [User]]))
-
-
-;(defn count-users []
-;  (get-in (first (select User (aggregate (count :*) :cnt))) [:cnt]))
-;
-;
-;; start with 0 entries
-;(expect 0 (count-users))
-;
-;; insert a new value
-;(expect (more->
-;          false nil?
-;          true (contains? :id))
-;  (ins User
-;    :email "user_test"
-;    :first_name "user_test"
-;    :last_name "user_test"
-;    :password "user_test"
-;    :date_joined (java.sql.Timestamp. (tc/to-long (time/now)))
-;    :is_active true
-;    :is_staff true
-;    :is_superuser false))
-;
-;; now we should have 1 entry
-;(expect 1 (count-users))

test/metabase/test_data/load.clj

@@ -20,7 +20,7 @@
 (def ^:private db-name "Test Database")
 (def ^:private org-name "Test Organization")
 (def ^:private test-db-filename
-  (delay (format "%s/t.db" (System/getProperty "user.dir"))))
+  (delay (format "%s/target/test-data" (System/getProperty "user.dir"))))
 (def ^:private test-db-connection-string
   (delay (format "file:%s;AUTO_SERVER=TRUE;DB_CLOSE_DELAY=-1" @test-db-filename)))
 
@@ -41,20 +41,20 @@
   []
   {:post [(map? %)]}
   (or (sel :one Database :name db-name)
-    (do (when-not (.exists (clojure.java.io/file (str @test-db-filename ".mv.db"))) ; only create + populate the test DB file if needed
-          (create-and-populate-tables))
-        (log/info "Creating new metabase Database object...")
-        (let [db (ins Database
-                   :organization_id (:id (test-org))
-                   :name db-name
-                   :engine :h2
-                   :details {:conn_str @test-db-connection-string})]
-          (log/info "Syncing Tables...")
-          (driver/sync-tables db)
-          (log/info "Adding Schema Metadata...")
-          (add-metadata!)
-          (log/info "Finished. Enjoy your test data <3")
-          db))))
+      (do (when-not (.exists (clojure.java.io/file (str @test-db-filename ".mv.db"))) ; only create + populate the test DB file if needed
+            (create-and-populate-tables))
+          (log/info "Creating new metabase Database object...")
+          (let [db (ins Database
+                     :organization_id (:id (test-org))
+                     :name db-name
+                     :engine :h2
+                     :details {:conn_str @test-db-connection-string})]
+            (log/info "Syncing Tables...")
+            (driver/sync-tables db)
+            (log/info "Adding Schema Metadata...")
+            (add-metadata!)
+            (log/info "Finished. Enjoy your test data <3")
+            db))))
 
 
 ;; ## Debugging/Interactive Development Functions
@@ -77,11 +77,11 @@
   "Binds `*test-db*` if not already bound to a Korma DB entity and executes BODY."
   [& body]
   `(if *test-db* (do ~@body)
-       (binding [*test-db* (create-db (h2 {:db @test-db-connection-string
-                                           :naming {:keys s/lower-case
-                                                    :fields s/upper-case}}))]
-         (log/info "CREATING H2 TEST DATABASE...")
-         ~@body)))
+                 (binding [*test-db* (create-db (h2 {:db @test-db-connection-string
+                                                     :naming {:keys s/lower-case
+                                                              :fields s/upper-case}}))]
+                   (log/info "CREATING H2 TEST DATABASE...")
+                   ~@body)))
 
 (defn- exec-sql
   "Execute raw SQL STATEMENTS against the test database."

test/metabase/test_utils.clj → test/metabase/test_setup.clj

-(ns metabase.test-utils ; TODO - rename to setup
+(ns metabase.test-setup
   "Functions that run before + after unit tests (setup DB, start web server, load test data)."
   (:require [clojure.java.io :as io]
             [clojure.tools.logging :as log]
             [expectations :refer :all]
-            [ring.adapter.jetty :as ring]
             (metabase [core :as core]
-                      [db :refer :all]
+                      [db :as db]
                       [test-data :refer :all])))
 
-(declare load-test-data
-         setup-test-db
-         start-jetty)
+(declare clear-test-db)
 
 ;; # SETTINGS
 
@@ -21,61 +18,39 @@
 
 ;; # FUNCTIONS THAT GET RUN ON TEST SUITE START / STOP
 
-(defn test-setup
+(defn test-startup
   {:expectations-options :before-run}
   []
-  ;; Disable debug logging since it clutters up our output
-  (.setLevel (org.apache.log4j.Logger/getLogger "metabase") org.apache.log4j.Level/INFO)
-  (setup-test-db)
-  (load-test-data)
-  (start-jetty))
+  (log/info "Starting up Metabase unit test runner")
+  ;; clear out any previous test data that's lying around
+  (clear-test-db)
+  ;; setup the db and migrate up to current schema
+  (db/setup-db :auto-migrate true)
+  ;; this causes the test data to be loaded
+  @test-db
+  ;; startup test web server
+  (core/start-jetty))
+
+
+(defn test-teardown
+  {:expectations-options :after-run}
+  []
+  (log/info "Shutting down Metabase unit test runner")
+  (core/stop-jetty))
+
 
 ;; ## DB Setup
 ;; WARNING: BY RUNNING ANY UNIT TESTS THAT REQUIRE THIS FILE OR BY RUNNING YOUR ENTIRE TEST SUITE YOU WILL EFFECTIVELY BE WIPING OUT YOUR DATABASE.
 ;; SETUP-DB DELETES YOUR DATABASE FILE, AND GETS RAN AUTOMATICALLY BY EXPECTATIONS. USE AT YOUR OWN RISK!
 
-(defn- setup-test-db
-  "Setup database schema."
+(defn- clear-test-db
+  "Delete the test db file if it's still lying around."
   []
-  (let [filename (-> (re-find #"file:(\w+\.db).*" (db-file)) second)] ; db-file is prefixed with "file:", so we strip that off
+  (let [filename (-> (re-find #"file:(\w+\.db).*" (db/db-file)) second)] ; db-file is prefixed with "file:", so we strip that off
     (map (fn [file-extension]                                         ; delete the database files, e.g. `metabase.db.h2.db`, `metabase.db.trace.db`, etc.
            (let [file (str filename file-extension)]
              (when (.exists (io/file file))
                (io/delete-file file))))
          [".h2.db"
           ".trace.db"
-          ".lock.db"]))
-  (log/info "tearing down database and resetting to empty schema")
-  (migrate (setup-jdbc-db) :down)
-  (log/info "setting up database and running all migrations")
-  (setup-db :auto-migrate true)
-  (log/info "database setup complete"))
-
-(defn- load-test-data []
-  @test-db)
-
-
-;; ## Jetty (Web) Server
-
-
-(def ^:private jetty-instance
-  (atom nil))
-
-(defn- start-jetty
-  "Start the Jetty web server."
-  []
-  (log/info "STARTING THE JETTY SERVER...")
-  (when-not @jetty-instance
-    (try (->> (ring/run-jetty core/app {:port 3000
-                                        :join? false}) ; detach the thread
-              (reset! jetty-instance))
-        (catch java.net.BindException e                ; assume server is already running if port's already bound
-          (log/warn "ALREADY RUNNING!")))))            ; e.g. if someone is running `lein ring server` locally. Tests should still work normally.
-
-(defn stop-jetty
-  "Stop the Jetty web server."
-  {:expectations-options :after-run}
-  []
-  (when @jetty-instance
-    (.stop ^org.eclipse.jetty.server.Server @jetty-instance)
-    (reset! jetty-instance nil)))
+          ".lock.db"])))