From 21829b3d2c1655c508394fc02871c0cec344a83b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Cam=20Sa=C3=BCl?= <cammsaul@gmail.com>
Date: Wed, 29 Jun 2016 19:33:03 -0700
Subject: [PATCH] Remove korma from load-from-h2 :honey_pot:
---
project.clj | 2 +-
resources/log4j.properties | 2 +-
src/metabase/cmd/load_from_h2.clj | 55 ++++++++++++++++---------------
src/metabase/core.clj | 2 +-
4 files changed, 32 insertions(+), 29 deletions(-)
diff --git a/project.clj b/project.clj
index e9f23523c12..178574815e4 100644
--- a/project.clj
+++ b/project.clj
@@ -43,12 +43,12 @@
"v2-rev300-1.22.0"]
[com.h2database/h2 "1.4.191"] ; embedded SQL database
[com.mattbertolini/liquibase-slf4j "2.0.0"] ; Java Migrations lib
+ [com.mchange/c3p0 "0.9.5.2"] ; connection pooling library
[com.novemberain/monger "3.0.2"] ; MongoDB Driver
[compojure "1.5.0"] ; HTTP Routing library built on Ring
[environ "1.0.3"] ; easy environment management
[hiccup "1.0.5"] ; HTML templating
[honeysql "0.6.3"] ; Transform Clojure data structures to SQL
- [korma "0.4.2"] ; SQL generation
[log4j/log4j "1.2.17" ; logging framework
:exclusions [javax.mail/mail
javax.jms/jms
diff --git a/resources/log4j.properties b/resources/log4j.properties
index d78f508692e..db7a8cd1e6e 100644
--- a/resources/log4j.properties
+++ b/resources/log4j.properties
@@ -23,4 +23,4 @@ log4j.logger.metabase.middleware=DEBUG
log4j.logger.metabase.query-processor=DEBUG
log4j.logger.metabase.sync-database=DEBUG
# c3p0 connection pools tend to log useless warnings way too often; only log actual errors
-log4j.logger.com.mchange.v2.resourcepool=ERROR
+log4j.logger.com.mchange=ERROR
diff --git a/src/metabase/cmd/load_from_h2.clj b/src/metabase/cmd/load_from_h2.clj
index b6a4f0e1fea..5201fa211b8 100644
--- a/src/metabase/cmd/load_from_h2.clj
+++ b/src/metabase/cmd/load_from_h2.clj
@@ -3,7 +3,6 @@
(:require [clojure.java.jdbc :as jdbc]
[clojure.set :as set]
[colorize.core :as color]
- [korma.core :as k]
[medley.core :as m]
[metabase.config :as config]
[metabase.db :as db]
@@ -82,32 +81,33 @@
"Active database connection to the target database we are loading into."
nil)
-(defn- insert-entity [e objs]
+;; TODO - `e` is a bad variable name! This should be something like `entity`
+(defn- insert-entity! [e objs]
(print (u/format-color 'blue "Transfering %d instances of %s..." (count objs) (:name e)))
(flush)
;; The connection closes prematurely on occasion when we're inserting thousands of rows at once. Break into smaller chunks so connection stays alive
(doseq [chunk (partition-all 300 objs)]
(print (color/blue \.))
(flush)
- (k/insert e (k/values (if (= e DashboardCard)
- ;; mini-HACK to fix h2 lowercasing these couple attributes
- ;; luckily this is the only place in our schema where we have camel case names
- (mapv #(set/rename-keys % {:sizex :sizeX, :sizey :sizeY}) chunk)
- chunk))))
+ (apply jdbc/insert! *db-conn* (:table e) (if (= e DashboardCard)
+ ;; mini-HACK to fix h2 lowercasing these couple attributes
+ ;; luckily this is the only place in our schema where we have camel case names
+ (mapv #(set/rename-keys % {:sizex :sizeX, :sizey :sizeY}) chunk)
+ chunk)))
(println (color/green "[OK]")))
-(defn- insert-self-referencing-entity [e objs]
+(defn- insert-self-referencing-entity! [e objs]
(let [self-ref-attr (condp = e
RawColumn :fk_target_column_id
Field :fk_target_field_id)
self-referencing (filter self-ref-attr objs)
others (set/difference (set objs) (set self-referencing))]
;; first insert the non-self-referencing objects
- (insert-entity e others)
+ (insert-entity! e others)
;; then insert the rest, which should be safe to insert now
- (insert-entity e self-referencing)))
+ (insert-entity! e self-referencing)))
-(defn- set-postgres-sequence-values []
+(defn- set-postgres-sequence-values! []
(print (u/format-color 'blue "Setting postgres sequence ids to proper values..."))
(flush)
(doseq [e (filter #(not (contains? entities-without-autoinc-ids %)) entities)
@@ -117,30 +117,33 @@
(jdbc/db-query-with-resultset *db-conn* [sql] :val))
(println (color/green "[OK]")))
-(defn load-from-h2
+(defn load-from-h2!
"Transfer data from existing H2 database to the newly created (presumably MySQL or Postgres) DB specified by env vars.
Intended as a tool for upgrading from H2 to a 'real' Database.
Defaults to using `@metabase.db/db-file` as the connection string."
[h2-connection-string-or-nil]
(db/setup-db)
- (let [h2-filename (or h2-connection-string-or-nil @metabase.db/db-file)]
+ (let [h2-filename (or h2-connection-string-or-nil @metabase.db/db-file)
+ target-db-spec (db/jdbc-details @db/db-connection-details)]
;; NOTE: would be nice to add `ACCESS_MODE_DATA=r` but it doesn't work with `AUTO_SERVER=TRUE`
;; connect to H2 database, which is what we are migrating from
(jdbc/with-db-connection [h2-conn (db/jdbc-details {:type :h2, :db (str h2-filename ";IFEXISTS=TRUE")})]
- (db/transaction
- (doseq [e entities
- :let [objs (->> (jdbc/query h2-conn [(str "SELECT * FROM " (:table e))])
- ;; we apply jdbc-clob->str to all row values because H2->Postgres
- ;; gets messed up if the value is left as a clob
- (map #(m/map-vals u/jdbc-clob->str %)))]
- :when (seq objs)]
- (if-not (contains? self-referencing-entities e)
- (insert-entity e objs)
- (insert-self-referencing-entity e objs)))))
+ (jdbc/with-db-transaction [target-db-conn target-db-spec]
+ (binding [*db-conn* target-db-conn]
+ (db/transaction
+ (doseq [e entities
+ :let [objs (->> (jdbc/query h2-conn [(str "SELECT * FROM " (name (:table e)))])
+ ;; we apply jdbc-clob->str to all row values because H2->Postgres
+ ;; gets messed up if the value is left as a clob
+ (map #(m/map-vals u/jdbc-clob->str %)))]
+ :when (seq objs)]
+ (if-not (contains? self-referencing-entities e)
+ (insert-entity! e objs)
+ (insert-self-referencing-entity! e objs)))))))
;; if we are loading into a postgres db then we need to update sequence nextvals
(when (= (config/config-str :mb-db-type) "postgres")
- (jdbc/with-db-transaction [targetdb-conn (db/jdbc-details @db/db-connection-details)]
- (binding [*db-conn* targetdb-conn]
- (set-postgres-sequence-values))))))
+ (jdbc/with-db-transaction [target-db-conn target-db-spec]
+ (binding [*db-conn* target-db-conn]
+ (set-postgres-sequence-values!))))))
diff --git a/src/metabase/core.clj b/src/metabase/core.clj
index 95cbd5b5613..ec9675bbe17 100644
--- a/src/metabase/core.clj
+++ b/src/metabase/core.clj
@@ -225,7 +225,7 @@
(db/migrate @db/db-connection-details (keyword direction)))
:load-from-h2 (fn [& [h2-connection-string-or-nil]]
(require 'metabase.cmd.load-from-h2)
- ((resolve 'metabase.cmd.load-from-h2/load-from-h2) h2-connection-string-or-nil))})
+ ((resolve 'metabase.cmd.load-from-h2/load-from-h2!) h2-connection-string-or-nil))})
(defn- run-cmd [cmd & args]
(let [f (or (cmd->fn cmd)
--
GitLab