From 3da9a2da39dd10f4b3587dc62e8bce95acfa9779 Mon Sep 17 00:00:00 2001
From: Cam Saul <cammsaul@gmail.com>
Date: Mon, 8 Apr 2019 19:01:24 -0700
Subject: [PATCH] Code cleanup :shower:
---
src/metabase/cmd/load_from_h2.clj | 128 +++++++++++++++++-------------
1 file changed, 72 insertions(+), 56 deletions(-)
diff --git a/src/metabase/cmd/load_from_h2.clj b/src/metabase/cmd/load_from_h2.clj
index ad31fe39199..740d877a9d3 100644
--- a/src/metabase/cmd/load_from_h2.clj
+++ b/src/metabase/cmd/load_from_h2.clj
@@ -57,11 +57,16 @@
[table :refer [Table]]
[task-history :refer [TaskHistory]]
[user :refer [User]]
- [view-log :refer [ViewLog]]]))
+ [view-log :refer [ViewLog]]]
+ [clojure.set :as set]
+ [toucan.db :as db])
+ (:import java.sql.SQLException))
(defn- println-ok [] (println (color/green "[OK]")))
-;;; -------------------------------------------------- Loading Data --------------------------------------------------
+(defn- dispatch-on-db-type [& _] (mdb/db-type))
+
+;;; ------------------------------------------ Models to Migrate (in order) ------------------------------------------
(def ^:private entities
"Entities in the order they should be serialized/deserialized. This is done so we make sure that we load load
@@ -102,6 +107,9 @@
;; above this line)
DataMigrations])
+
+;;; --------------------------------------------- H2 Connection Options ----------------------------------------------
+
(defn- add-file-prefix-if-needed [connection-string-or-filename]
(if (str/starts-with? connection-string-or-filename "file:")
connection-string-or-filename
@@ -112,31 +120,28 @@
(mdb/jdbc-details {:type :h2, :db (str h2-filename ";IFEXISTS=TRUE")})))
-(defn- insert-entity! [target-db-conn entity objs]
- (print (u/format-color 'blue "Transfering %d instances of %s..." (count objs) (:name entity))) ; TODO - I don't think the print+flush is working as intended :/
+;;; ------------------------------------------- Fetching & Inserting Rows --------------------------------------------
+
+(defn- insert-entity! [target-db-conn {:keys [table], :as entity} objs]
+ ;; TODO - I don't think the print+flush is working as intended :/
+ (print (u/format-color 'blue "Transfering %d instances of %s..." (count objs) (:name entity)))
(flush)
- (let [ks (keys (first objs))
- ;; 1) `:sizeX` and `:sizeY` come out of H2 as `:sizex` and `:sizey` because of automatic lowercasing; fix the
- ;; names of these before putting into the new DB
- ;; 2) Need to wrap the column names in quotes because Postgres automatically lowercases unquoted identifiers
- quote-char (case (config/config-kw :mb-db-type)
- :postgres \"
- :mysql \`)
- cols (for [k ks]
- (str quote-char (name (case k
- :sizex :sizeX
- :sizey :sizeY
- k)) quote-char))]
+ ;; 1) `:sizeX` and `:sizeY` come out of H2 as `:sizex` and `:sizey` because of automatic lowercasing; fix the
+ ;; names of these before putting into the new DB
+ ;; 2) Need to wrap the column names in quotes because Postgres automatically lowercases unquoted identifiers
+ (let [ks (keys (set/rename-keys (first objs) {:sizex :sizeX, :sizey :sizeY}))
+ cols (for [k ks]
+ ((db/quote-fn) (name k)))]
;; The connection closes prematurely on occasion when we're inserting thousands of rows at once. Break into
;; smaller chunks so connection stays alive
- (doseq [chunk (partition-all 300 objs)]
+ (doseq [chunk (partition-all 300 objs)
+ :let [vals (for [row chunk]
+ (map row ks))]]
(print (color/blue \.))
(flush)
(try
- (jdbc/insert-multi! target-db-conn (:table entity) cols (for [row chunk]
- (map row ks)))
-
- (catch java.sql.SQLException e
+ (jdbc/insert-multi! target-db-conn table cols vals)
+ (catch SQLException e
(jdbc/print-sql-exception-chain e)
(throw e)))))
(println-ok))
@@ -153,7 +158,16 @@
;;; ---------------------------------------- Enabling / Disabling Constraints ----------------------------------------
-(defn- disable-db-constraints:postgres! [target-db-conn]
+(defmulti ^:private disable-db-constraints!
+ {:arglists '([target-db-conn])}
+ dispatch-on-db-type)
+
+(defmulti ^:private reenable-db-constraints!
+ {:arglists '([target-db-conn])}
+ dispatch-on-db-type)
+
+
+(defmethod disable-db-constraints! :postgres [target-db-conn]
;; make all of our FK constraints deferrable. This only works on Postgres 9.4+ (December 2014)! (There's no pressing
;; reason to turn these back on at the conclusion of this script. It makes things more complicated since it doesn't
;; work if done inside the same transaction.)
@@ -166,49 +180,41 @@
;; now enable constraint deferring for the duration of the transaction
(jdbc/execute! target-db-conn ["SET CONSTRAINTS ALL DEFERRED"]))
+(defmethod reenable-db-constraints! :postgres [_]) ; no-op
+
-(defn- disable-db-constraints:mysql! [target-db-conn]
+(defmethod disable-db-constraints! :mysql [target-db-conn]
(jdbc/execute! target-db-conn ["SET FOREIGN_KEY_CHECKS=0"]))
-;; For MySQL we need to reënable FK checks when we're done
-(defn- reënable-db-constraints:mysql! [target-db-conn]
+;; For MySQL we need to re-enable FK checks when we're done
+(defmethod reenable-db-constraints! :mysql [target-db-conn]
(jdbc/execute! target-db-conn ["SET FOREIGN_KEY_CHECKS=1"]))
-(defn- disable-db-constraints! [target-db-conn]
- (println (u/format-color 'blue "Temporarily disabling DB constraints..."))
- ((case (mdb/db-type)
- :postgres disable-db-constraints:postgres!
- :mysql disable-db-constraints:mysql!) target-db-conn)
- (println-ok))
-
-(defn- reënable-db-constraints-if-needed! [target-db-conn]
- (when (= (mdb/db-type) :mysql)
- (println (u/format-color 'blue "Reënabling DB constraints..."))
- (reënable-db-constraints:mysql! target-db-conn)
- (println-ok)))
-
-
-;;; ---------------------------------------- Fixing Postgres Sequence Values -----------------------------------------
+;;; --------------------------------------------- Fixing Sequence Values ---------------------------------------------
(def ^:private entities-without-autoinc-ids
"Entities that do NOT use an auto incrementing ID column."
#{Setting Session DataMigrations})
-(defn- set-postgres-sequence-values-if-needed!
- "When loading data into a Postgres DB, update the sequence nextvals."
- []
- (when (= (mdb/db-type) :postgres)
- (jdbc/with-db-transaction [target-db-conn (mdb/jdbc-details)]
- (println (u/format-color 'blue "Setting postgres sequence ids to proper values..."))
- (doseq [e entities
- :when (not (contains? entities-without-autoinc-ids e))
- :let [table-name (name (:table e))
- seq-name (str table-name "_id_seq")
- sql (format "SELECT setval('%s', COALESCE((SELECT MAX(id) FROM %s), 1), true) as val"
- seq-name (name table-name))]]
- (jdbc/db-query-with-resultset target-db-conn [sql] :val))
- (println-ok))))
+(defmulti ^:private update-sequence-values!
+ {:arglists '([])}
+ dispatch-on-db-type)
+
+(defmethod update-sequence-values! :mysql []) ; no-op
+
+;; Update the sequence nextvals.
+(defmethod update-sequence-values! :postgres []
+ (jdbc/with-db-transaction [target-db-conn (mdb/jdbc-details)]
+ (println (u/format-color 'blue "Setting postgres sequence ids to proper values..."))
+ (doseq [e entities
+ :when (not (contains? entities-without-autoinc-ids e))
+ :let [table-name (name (:table e))
+ seq-name (str table-name "_id_seq")
+ sql (format "SELECT setval('%s', COALESCE((SELECT MAX(id) FROM %s), 1), true) as val"
+ seq-name (name table-name))]]
+ (jdbc/db-query-with-resultset target-db-conn [sql] :val))
+ (println-ok)))
;;; --------------------------------------------------- Public Fns ---------------------------------------------------
@@ -220,10 +226,20 @@
Defaults to using `@metabase.db/db-file` as the connection string."
[h2-connection-string-or-nil]
(mdb/setup-db!)
+
(jdbc/with-db-transaction [target-db-conn (mdb/jdbc-details)]
(jdbc/db-set-rollback-only! target-db-conn)
+
+ (println (u/format-color 'blue "Temporarily disabling DB constraints..."))
(disable-db-constraints! target-db-conn)
+ (println-ok)
+
(load-data! target-db-conn h2-connection-string-or-nil)
- (reënable-db-constraints-if-needed! (mdb/jdbc-details))
+
+ (println (u/format-color 'blue "Re-enabling DB constraints..."))
+ (reenable-db-constraints! target-db-conn)
+ (println-ok)
+
(jdbc/db-unset-rollback-only! target-db-conn))
- (set-postgres-sequence-values-if-needed!))
+
+ (update-sequence-values!))
--
GitLab