Remove korma from load-from-h2

project.clj

@@ -43,12 +43,12 @@
                   "v2-rev300-1.22.0"]
                  [com.h2database/h2 "1.4.191"]                        ; embedded SQL database
                  [com.mattbertolini/liquibase-slf4j "2.0.0"]          ; Java Migrations lib
+                 [com.mchange/c3p0 "0.9.5.2"]                         ; connection pooling library
                  [com.novemberain/monger "3.0.2"]                     ; MongoDB Driver
                  [compojure "1.5.0"]                                  ; HTTP Routing library built on Ring
                  [environ "1.0.3"]                                    ; easy environment management
                  [hiccup "1.0.5"]                                     ; HTML templating
                  [honeysql "0.6.3"]                                   ; Transform Clojure data structures to SQL
-                 [korma "0.4.2"]                                      ; SQL generation
                  [log4j/log4j "1.2.17"                                ; logging framework
                   :exclusions [javax.mail/mail
                                javax.jms/jms

resources/log4j.properties

@@ -23,4 +23,4 @@ log4j.logger.metabase.middleware=DEBUG
 log4j.logger.metabase.query-processor=DEBUG
 log4j.logger.metabase.sync-database=DEBUG
 # c3p0 connection pools tend to log useless warnings way too often; only log actual errors
-log4j.logger.com.mchange.v2.resourcepool=ERROR
+log4j.logger.com.mchange=ERROR

src/metabase/cmd/load_from_h2.clj

@@ -3,7 +3,6 @@
   (:require [clojure.java.jdbc :as jdbc]
             [clojure.set :as set]
             [colorize.core :as color]
-            [korma.core :as k]
             [medley.core :as m]
             [metabase.config :as config]
             [metabase.db :as db]
@@ -82,32 +81,33 @@
   "Active database connection to the target database we are loading into."
   nil)
 
-(defn- insert-entity [e objs]
+;; TODO - `e` is a bad variable name! This should be something like `entity`
+(defn- insert-entity! [e objs]
   (print (u/format-color 'blue "Transfering %d instances of %s..." (count objs) (:name e)))
   (flush)
   ;; The connection closes prematurely on occasion when we're inserting thousands of rows at once. Break into smaller chunks so connection stays alive
   (doseq [chunk (partition-all 300 objs)]
     (print (color/blue \.))
     (flush)
-    (k/insert e (k/values (if (= e DashboardCard)
-                            ;; mini-HACK to fix h2 lowercasing these couple attributes
-                            ;; luckily this is the only place in our schema where we have camel case names
-                            (mapv #(set/rename-keys % {:sizex :sizeX, :sizey :sizeY}) chunk)
-                            chunk))))
+    (apply jdbc/insert! *db-conn* (:table e) (if (= e DashboardCard)
+                                               ;; mini-HACK to fix h2 lowercasing these couple attributes
+                                               ;; luckily this is the only place in our schema where we have camel case names
+                                               (mapv #(set/rename-keys % {:sizex :sizeX, :sizey :sizeY}) chunk)
+                                               chunk)))
   (println (color/green "[OK]")))
 
-(defn- insert-self-referencing-entity [e objs]
+(defn- insert-self-referencing-entity! [e objs]
   (let [self-ref-attr    (condp = e
                            RawColumn :fk_target_column_id
                            Field     :fk_target_field_id)
         self-referencing (filter self-ref-attr objs)
         others           (set/difference (set objs) (set self-referencing))]
     ;; first insert the non-self-referencing objects
-    (insert-entity e others)
+    (insert-entity! e others)
     ;; then insert the rest, which should be safe to insert now
-    (insert-entity e self-referencing)))
+    (insert-entity! e self-referencing)))
 
-(defn- set-postgres-sequence-values []
+(defn- set-postgres-sequence-values! []
   (print (u/format-color 'blue "Setting postgres sequence ids to proper values..."))
   (flush)
   (doseq [e    (filter #(not (contains? entities-without-autoinc-ids %)) entities)
@@ -117,30 +117,33 @@
     (jdbc/db-query-with-resultset *db-conn* [sql] :val))
   (println (color/green "[OK]")))
 
-(defn load-from-h2
+(defn load-from-h2!
   "Transfer data from existing H2 database to the newly created (presumably MySQL or Postgres) DB specified by env vars.
    Intended as a tool for upgrading from H2 to a 'real' Database.
 
    Defaults to using `@metabase.db/db-file` as the connection string."
   [h2-connection-string-or-nil]
   (db/setup-db)
-  (let [h2-filename (or h2-connection-string-or-nil @metabase.db/db-file)]
+  (let [h2-filename    (or h2-connection-string-or-nil @metabase.db/db-file)
+        target-db-spec (db/jdbc-details @db/db-connection-details)]
     ;; NOTE: would be nice to add `ACCESS_MODE_DATA=r` but it doesn't work with `AUTO_SERVER=TRUE`
     ;; connect to H2 database, which is what we are migrating from
     (jdbc/with-db-connection [h2-conn (db/jdbc-details {:type :h2, :db (str h2-filename ";IFEXISTS=TRUE")})]
-      (db/transaction
-        (doseq [e     entities
-                :let  [objs (->> (jdbc/query h2-conn [(str "SELECT * FROM " (:table e))])
-                                 ;; we apply jdbc-clob->str to all row values because H2->Postgres
-                                 ;; gets messed up if the value is left as a clob
-                                 (map #(m/map-vals u/jdbc-clob->str %)))]
-                :when (seq objs)]
-          (if-not (contains? self-referencing-entities e)
-            (insert-entity e objs)
-            (insert-self-referencing-entity e objs)))))
+      (jdbc/with-db-transaction [target-db-conn target-db-spec]
+        (binding [*db-conn* target-db-conn]
+          (db/transaction
+           (doseq [e     entities
+                   :let  [objs (->> (jdbc/query h2-conn [(str "SELECT * FROM " (name (:table e)))])
+                                    ;; we apply jdbc-clob->str to all row values because H2->Postgres
+                                    ;; gets messed up if the value is left as a clob
+                                    (map #(m/map-vals u/jdbc-clob->str %)))]
+                   :when (seq objs)]
+             (if-not (contains? self-referencing-entities e)
+               (insert-entity! e objs)
+               (insert-self-referencing-entity! e objs)))))))
 
     ;; if we are loading into a postgres db then we need to update sequence nextvals
     (when (= (config/config-str :mb-db-type) "postgres")
-      (jdbc/with-db-transaction [targetdb-conn (db/jdbc-details @db/db-connection-details)]
-        (binding [*db-conn* targetdb-conn]
-          (set-postgres-sequence-values))))))
+      (jdbc/with-db-transaction [target-db-conn target-db-spec]
+        (binding [*db-conn* target-db-conn]
+          (set-postgres-sequence-values!))))))

src/metabase/core.clj

@@ -225,7 +225,7 @@
                    (db/migrate @db/db-connection-details (keyword direction)))
    :load-from-h2 (fn [& [h2-connection-string-or-nil]]
                    (require 'metabase.cmd.load-from-h2)
-                   ((resolve 'metabase.cmd.load-from-h2/load-from-h2) h2-connection-string-or-nil))})
+                   ((resolve 'metabase.cmd.load-from-h2/load-from-h2!) h2-connection-string-or-nil))})
 
 (defn- run-cmd [cmd & args]
   (let [f (or (cmd->fn cmd)