SQL Server Driver

project.clj

@@ -41,13 +41,13 @@
                                com.sun.jmx/jmxri]]
                  [medley "0.7.0"]                                     ; lightweight lib of useful functions
                  [mysql/mysql-connector-java "5.1.37"]                ; MySQL JDBC driver
-                 [net.sourceforge.jtds/jtds "1.3.1"]                  ; SQL Server driver
                  [org.liquibase/liquibase-core "3.4.1"]               ; migration management (Java lib)
                  [org.slf4j/slf4j-log4j12 "1.7.12"]
                  [org.yaml/snakeyaml "1.16"]                          ; YAML parser (required by liquibase)
                  [postgresql "9.3-1102.jdbc41"]                       ; Postgres driver
                  [ring/ring-jetty-adapter "1.4.0"]                    ; Ring adapter using Jetty webserver (used to run a Ring server for unit tests)
                  [ring/ring-json "0.4.0"]                             ; Ring middleware for reading/writing JSON automatically
+                 [microsoft/sqljdbc4 "4.2"]
                  [stencil "0.5.0"]                                    ; Mustache templates for Clojure
                  [swiss-arrows "1.0.0"]]                              ; 'Magic wand' macro -<>, etc.
   :plugins [[lein-environ "1.0.0"]                                    ; easy access to environment variables
@@ -76,6 +76,7 @@
                              [lein-environ "1.0.0"]                   ; Specify env-vars in project.clj
                              [lein-expectations "0.0.8"]              ; run unit tests with 'lein expectations'
                              [lein-instant-cheatsheet "2.1.4"]        ; use awesome instant cheatsheet created by yours truly w/ 'lein instant-cheatsheet'
+                             [lein-localrepo "0.5.3"]
                              [michaelblume/lein-marginalia "0.9.0"]   ; generate documentation with 'lein marg'
                              [refactor-nrepl "2.0.0-SNAPSHOT"]]       ; support for advanced refactoring in Emacs/LightTable
                    :global-vars {*warn-on-reflection* true}           ; Emit warnings on all reflection calls

src/metabase/driver.clj

@@ -336,13 +336,18 @@
 
 ;; ## Implementation-Agnostic Driver API
 
+(def ^:private ^:const can-connect-timeout-ms
+  "Consider `can-connect?`/`can-connect-with-details?` to have failed after this many milliseconds."
+  1000)
+
 (defn can-connect?
   "Check whether we can connect to DATABASE and perform a basic query (such as `SELECT 1`)."
   [database]
   {:pre [(map? database)]}
   (let [driver (engine->driver (:engine database))]
     (try
-      ((:can-connect? driver) (:details database))
+      (u/with-timeout can-connect-timeout-ms
+        ((:can-connect? driver) (:details database)))
       (catch Throwable e
         (log/error "Failed to connect to database:" (.getMessage e))
         false))))
@@ -359,7 +364,8 @@
          (map? details-map)]}
   (let [{:keys [humanize-connection-error-message], :as driver} (engine->driver engine)]
     (try
-      ((:can-connect? driver) details-map)
+      (u/with-timeout can-connect-timeout-ms
+        ((:can-connect? driver) details-map))
       (catch Throwable e
         (log/error "Failed to connect to database:" (.getMessage e))
         (when rethrow-exceptions

src/metabase/driver/sqlserver.clj

@@ -4,18 +4,19 @@
             [korma.sql.utils :as utils]
             [metabase.driver :refer [defdriver]]
             [metabase.driver.generic-sql :refer [sql-driver]])
-  (:import net.sourceforge.jtds.jdbc.Driver)) ; need to import this in order to load JDBC driver
+  ;; (:import net.sourceforge.jtds.jdbc.Driver)
+  ) ; need to import this in order to load JDBC driver
 
-(defn- connection-details->spec [details-map]
-  (assoc (kdb/mssql details-map)
-         :classname   "net.sourceforge.jtds.jdbc.Driver"
-         :subprotocol "jtds:sqlserver"))
+;; (defn- connection-details->spec [details-map]
+;;   (assoc (kdb/mssql details-map)
+;;          :classname   "net.sourceforge.jtds.jdbc.Driver"
+;;          :subprotocol "jtds:sqlserver"))
 
 (def ^:private ^:const column->base-type
   "See [this page](https://msdn.microsoft.com/en-us/library/ms187752.aspx) for details."
   {:bigint           :BigIntegerField
    :binary           :UnknownField
-   :bit              :UnknownField
+   :bit              :BooleanField     ; actually this is 1 / 0 instead of true / false ...
    :char             :CharField
    :cursor           :UnknownField
    :date             :DateField
@@ -81,7 +82,7 @@
                    :sql-string-length-fn      :LEN
                    ;; :ignored-schemas           #{"sys" "INFORMATION_SCHEMA"}
                    :column->base-type         column->base-type
-                   :connection-details->spec  connection-details->spec
+                   :connection-details->spec  kdb/mssql ; connection-details->spec
                    :date                      date
                    :date-interval             date-interval
                    :unix-timestamp->timestamp unix-timestamp->timestamp})

src/metabase/util.clj

@@ -279,4 +279,20 @@
       (->> (map str (.getStackTrace e))
            (filterv (partial re-find #"metabase"))))))
 
+(defmacro with-timeout
+  "Run BODY in a `future` and throw an exception if it fails to complete after TIMEOUT-MS."
+  [timeout-ms & body]
+  `(let [future# (future ~@body)
+         result# (deref future# ~timeout-ms :timeout)]
+     (when (= result# :timeout)
+       (throw (Exception. (format "Timed out after %d milliseconds." ~timeout-ms))))
+     result#))
+
+(defmacro pfor
+  "Parallel form of `for`."
+  [[binding collection] & body]
+  `(pmap (fn [~binding]
+           ~@body)
+         ~(seq collection)))
+
 (require-dox-in-this-namespace)

test/metabase/driver/query_processor_test.clj

 (ns metabase.driver.query-processor-test
   "Query processing tests that can be ran between any of the available drivers, and should give the same results."
-  (:require [clojure.walk :as walk]
+  (:require [clojure.java.jdbc :as jdbc]
+            [clojure.walk :as walk]
             [expectations :refer :all]
             [metabase.db :refer :all]
             [metabase.driver :as driver]
             [metabase.driver.query-processor :refer :all]
             (metabase.models [field :refer [Field]]
                              [table :refer [Table]])
-            [metabase.test.data :refer :all]
             (metabase.test.data [dataset-definitions :as defs]
                                 [datasets :as datasets :refer [*dataset*]]
                                 [interface :refer [create-database-definition]])
+            [metabase.test.data :refer :all]
             [metabase.test.util.q :refer [Q]]
             [metabase.util :as u]))
 
@@ -825,6 +826,14 @@
      filter = category_id->categories.id 8))
 
 
+(defn x []
+  (try (Q dataset tupac-sightings use sqlserver
+          return rows of sightings
+          limit 10)
+       (catch java.sql.SQLException e
+         (jdbc/print-sql-exception-chain e))))
+
+
 ;; THE 10 MOST RECENT TUPAC SIGHTINGS (!)
 ;; (What he was doing when we saw him, sighting ID)
 ;; Check that we can include an FK field in the :fields clause

test/metabase/test/data.clj

@@ -183,9 +183,13 @@
         dbdef  (map->DatabaseDefinition (assoc dbdef :short-lived? true))]
     (try
       (binding [*sel-disable-logging* true]
+        (println "<remove-database!>")
         (remove-database! loader dbdef)
+        (println "</remove-database!>")
+        (println "<get-or-create-database!>")
         (let [db (-> (get-or-create-database! loader dbdef)
                      temp-db-add-getter-delay)]
+          (println "</get-or-create-database!>")
           (assert db)
           (assert (exists? Database :id (:id db)))
           (binding [*temp-db*             db
@@ -193,7 +197,11 @@
             (f db))))
       (finally
         (binding [*sel-disable-logging* true]
-          (remove-database! loader dbdef))))))
+          (println "<finally>")
+          (try
+            (remove-database! loader dbdef)
+            (catch Throwable _))
+          (println "</finally>"))))))
 
 
 (defmacro with-temp-db

test/metabase/test/data/datasets.clj

@@ -13,7 +13,8 @@
                                 [h2 :as h2]
                                 [mongo :as mongo]
                                 [mysql :as mysql]
-                                [postgres :as postgres])
+                                [postgres :as postgres]
+                                [sqlserver :as sqlserver])
             [metabase.util :as u]))
 
 ;; # IDataset
@@ -163,14 +164,27 @@
           :sum-field-type (constantly :BigIntegerField)}))
 
 
+;;; ### SQLServer
+
+(defrecord SQLServerDriverData [dbpromise])
+
+(extend SQLServerDriverData
+  IDataset
+  (merge GenericSQLIDatasetMixin
+         {:dataset-loader (fn [_]
+                            (sqlserver/dataset-loader))
+          :sum-field-type (constantly :IntegerField)}))
+
+
 ;; # Concrete Instances
 
 (def dataset-name->dataset
   "Map of dataset keyword name -> dataset instance (i.e., an object that implements `IDataset`)."
-  {:mongo    (MongoDriverData.)
-   :h2       (H2DriverData. (promise))
-   :postgres (PostgresDriverData. (promise))
-   :mysql    (MySQLDriverData. (promise))})
+  {:mongo     (MongoDriverData.)
+   :h2        (H2DriverData.        (promise))
+   :postgres  (PostgresDriverData.  (promise))
+   :mysql     (MySQLDriverData.     (promise))
+   :sqlserver (SQLServerDriverData. (promise))})
 
 (def ^:const all-valid-dataset-names
   "Set of names of all valid datasets."

test/metabase/test/data/generic_sql.clj

@@ -2,7 +2,8 @@
   "Common functionality for various Generic SQL dataset loaders."
   (:require [clojure.tools.logging :as log]
             [korma.core :as k]
-            [metabase.test.data.interface :as i])
+            [metabase.test.data.interface :as i]
+            [metabase.util :as u])
   (:import (metabase.test.data.interface DatabaseDefinition
                                          TableDefinition)))
 
@@ -32,7 +33,9 @@
     "Given a `Field.base_type`, return the SQL type we should use for that column when creating a DB."))
 
 
-(defn create-physical-table! [dataset-loader database-definition {:keys [table-name field-definitions], :as table-definition}]
+(defn create-physical-table! [dataset-loader {:keys [database-name], :as database-definition} {:keys [table-name field-definitions], :as table-definition}
+                              & {:keys [table-name-qualification]
+                                 :or   {table-name-qualification :none}}]
   ;; Drop the table if it already exists
   (i/drop-physical-table! dataset-loader database-definition table-definition)
 
@@ -41,7 +44,9 @@
     (let [quot (partial quote-name dataset-loader)
           pk-field-name (quot (pk-field-name dataset-loader))]
       (format "CREATE TABLE %s (%s, %s %s, PRIMARY KEY (%s));"
-              (quot table-name)
+              (case table-name-qualification
+                :database (format "%s.%s" (quot database-name) (quot table-name))
+                :none     (quot table-name))
               (->> field-definitions
                    (map (fn [{:keys [field-name base-type]}]
                           (format "%s %s" (quot field-name) (field-base-type->sql-type dataset-loader base-type))))
@@ -78,10 +83,16 @@
 
 (defn load-table-data! [dataset-loader database-definition table-definition]
   (let [rows              (:rows table-definition)
-        fields-for-insert (map :field-name (:field-definitions table-definition))]
-    (-> (korma-entity dataset-loader database-definition table-definition)
-        (k/insert (k/values (->> (for [row rows]
-                                   (for [v row]
-                                     (if (instance? java.util.Date v) (java.sql.Timestamp. (.getTime ^java.util.Date v))
-                                         v)))
-                                 (map (partial zipmap fields-for-insert))))))))
+        fields-for-insert (mapv :field-name (:field-definitions table-definition))
+        entity            (korma-entity dataset-loader database-definition table-definition)]
+    ;; Insert groups of 200 rows at a time
+    ;; otherwise SQL Server will be *very* snippy if we try to run queries with too many parameters in them
+    ;; We can prepare the queries in parallel, however, and save ourselves a bit of time
+    (doseq [query (u/pfor [group (partition-all 200 rows)]
+                    (-> (k/insert* entity)
+                        (k/values (mapv (partial zipmap fields-for-insert)
+                                        (for [row group]
+                                          (for [v row]
+                                            (if (instance? java.util.Date v) (java.sql.Timestamp. (.getTime ^java.util.Date v))
+                                                v)))))))]
+      (k/exec query))))

test/metabase/test/util/q.clj

@@ -63,8 +63,8 @@
 
 (defn resolve-dataset [dataset]
   (var-get (core/or (resolve dataset)
-                       (ns-resolve 'metabase.test.data.dataset-definitions dataset)
-                       (throw (Exception. (format "Don't know how to find dataset '%s'." dataset))))))
+                    (ns-resolve 'metabase.test.data.dataset-definitions dataset)
+                    (throw (Exception. (format "Don't know how to find dataset '%s'." dataset))))))
 
 
 ;;; # DSL KEYWORD MACROS