Skip to content
Snippets Groups Projects
Unverified Commit 3da9a2da authored by Cam Saul's avatar Cam Saul
Browse files

Code cleanup :shower:

parent b564a5ad
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
src/metabase/cmd/load_from_h2.clj
+ 72
− 56
View file @ 3da9a2da
......@@ -57,11 +57,16 @@
[table :refer [Table]]
[task-history :refer [TaskHistory]]
[user :refer [User]]
[view-log :refer [ViewLog]]]))
[view-log :refer [ViewLog]]]
[clojure.set :as set]
[toucan.db :as db])
(:import java.sql.SQLException))
(defn- println-ok [] (println (color/green "[OK]")))
;;; -------------------------------------------------- Loading Data --------------------------------------------------
(defn- dispatch-on-db-type [& _] (mdb/db-type))
;;; ------------------------------------------ Models to Migrate (in order) ------------------------------------------
(def ^:private entities
"Entities in the order they should be serialized/deserialized. This is done so we make sure that we load load
......@@ -102,6 +107,9 @@
;; above this line)
DataMigrations])
;;; --------------------------------------------- H2 Connection Options ----------------------------------------------
(defn- add-file-prefix-if-needed [connection-string-or-filename]
(if (str/starts-with? connection-string-or-filename "file:")
connection-string-or-filename
......@@ -112,31 +120,28 @@
(mdb/jdbc-details {:type :h2, :db (str h2-filename ";IFEXISTS=TRUE")})))
(defn- insert-entity! [target-db-conn entity objs]
(print (u/format-color 'blue "Transfering %d instances of %s..." (count objs) (:name entity))) ; TODO - I don't think the print+flush is working as intended :/
;;; ------------------------------------------- Fetching & Inserting Rows --------------------------------------------
(defn- insert-entity! [target-db-conn {:keys [table], :as entity} objs]
;; TODO - I don't think the print+flush is working as intended :/
(print (u/format-color 'blue "Transfering %d instances of %s..." (count objs) (:name entity)))
(flush)
(let [ks (keys (first objs))
;; 1) `:sizeX` and `:sizeY` come out of H2 as `:sizex` and `:sizey` because of automatic lowercasing; fix the
;; names of these before putting into the new DB
;; 2) Need to wrap the column names in quotes because Postgres automatically lowercases unquoted identifiers
quote-char (case (config/config-kw :mb-db-type)
:postgres \"
:mysql \`)
cols (for [k ks]
(str quote-char (name (case k
:sizex :sizeX
:sizey :sizeY
k)) quote-char))]
;; 1) `:sizeX` and `:sizeY` come out of H2 as `:sizex` and `:sizey` because of automatic lowercasing; fix the
;; names of these before putting into the new DB
;; 2) Need to wrap the column names in quotes because Postgres automatically lowercases unquoted identifiers
(let [ks (keys (set/rename-keys (first objs) {:sizex :sizeX, :sizey :sizeY}))
cols (for [k ks]
((db/quote-fn) (name k)))]
;; The connection closes prematurely on occasion when we're inserting thousands of rows at once. Break into
;; smaller chunks so connection stays alive
(doseq [chunk (partition-all 300 objs)]
(doseq [chunk (partition-all 300 objs)
:let [vals (for [row chunk]
(map row ks))]]
(print (color/blue \.))
(flush)
(try
(jdbc/insert-multi! target-db-conn (:table entity) cols (for [row chunk]
(map row ks)))
(catch java.sql.SQLException e
(jdbc/insert-multi! target-db-conn table cols vals)
(catch SQLException e
(jdbc/print-sql-exception-chain e)
(throw e)))))
(println-ok))
......@@ -153,7 +158,16 @@
;;; ---------------------------------------- Enabling / Disabling Constraints ----------------------------------------
(defn- disable-db-constraints:postgres! [target-db-conn]
(defmulti ^:private disable-db-constraints!
{:arglists '([target-db-conn])}
dispatch-on-db-type)
(defmulti ^:private reenable-db-constraints!
{:arglists '([target-db-conn])}
dispatch-on-db-type)
(defmethod disable-db-constraints! :postgres [target-db-conn]
;; make all of our FK constraints deferrable. This only works on Postgres 9.4+ (December 2014)! (There's no pressing
;; reason to turn these back on at the conclusion of this script. It makes things more complicated since it doesn't
;; work if done inside the same transaction.)
......@@ -166,49 +180,41 @@
;; now enable constraint deferring for the duration of the transaction
(jdbc/execute! target-db-conn ["SET CONSTRAINTS ALL DEFERRED"]))
(defmethod reenable-db-constraints! :postgres [_]) ; no-op
(defn- disable-db-constraints:mysql! [target-db-conn]
(defmethod disable-db-constraints! :mysql [target-db-conn]
(jdbc/execute! target-db-conn ["SET FOREIGN_KEY_CHECKS=0"]))
;; For MySQL we need to reënable FK checks when we're done
(defn- reënable-db-constraints:mysql! [target-db-conn]
;; For MySQL we need to re-enable FK checks when we're done
(defmethod reenable-db-constraints! :mysql [target-db-conn]
(jdbc/execute! target-db-conn ["SET FOREIGN_KEY_CHECKS=1"]))
(defn- disable-db-constraints! [target-db-conn]
(println (u/format-color 'blue "Temporarily disabling DB constraints..."))
((case (mdb/db-type)
:postgres disable-db-constraints:postgres!
:mysql disable-db-constraints:mysql!) target-db-conn)
(println-ok))
(defn- reënable-db-constraints-if-needed! [target-db-conn]
(when (= (mdb/db-type) :mysql)
(println (u/format-color 'blue "Reënabling DB constraints..."))
(reënable-db-constraints:mysql! target-db-conn)
(println-ok)))
;;; ---------------------------------------- Fixing Postgres Sequence Values -----------------------------------------
;;; --------------------------------------------- Fixing Sequence Values ---------------------------------------------
(def ^:private entities-without-autoinc-ids
"Entities that do NOT use an auto incrementing ID column."
#{Setting Session DataMigrations})
(defn- set-postgres-sequence-values-if-needed!
"When loading data into a Postgres DB, update the sequence nextvals."
[]
(when (= (mdb/db-type) :postgres)
(jdbc/with-db-transaction [target-db-conn (mdb/jdbc-details)]
(println (u/format-color 'blue "Setting postgres sequence ids to proper values..."))
(doseq [e entities
:when (not (contains? entities-without-autoinc-ids e))
:let [table-name (name (:table e))
seq-name (str table-name "_id_seq")
sql (format "SELECT setval('%s', COALESCE((SELECT MAX(id) FROM %s), 1), true) as val"
seq-name (name table-name))]]
(jdbc/db-query-with-resultset target-db-conn [sql] :val))
(println-ok))))
(defmulti ^:private update-sequence-values!
{:arglists '([])}
dispatch-on-db-type)
(defmethod update-sequence-values! :mysql []) ; no-op
;; Update the sequence nextvals.
(defmethod update-sequence-values! :postgres []
(jdbc/with-db-transaction [target-db-conn (mdb/jdbc-details)]
(println (u/format-color 'blue "Setting postgres sequence ids to proper values..."))
(doseq [e entities
:when (not (contains? entities-without-autoinc-ids e))
:let [table-name (name (:table e))
seq-name (str table-name "_id_seq")
sql (format "SELECT setval('%s', COALESCE((SELECT MAX(id) FROM %s), 1), true) as val"
seq-name (name table-name))]]
(jdbc/db-query-with-resultset target-db-conn [sql] :val))
(println-ok)))
;;; --------------------------------------------------- Public Fns ---------------------------------------------------
......@@ -220,10 +226,20 @@
Defaults to using `@metabase.db/db-file` as the connection string."
[h2-connection-string-or-nil]
(mdb/setup-db!)
(jdbc/with-db-transaction [target-db-conn (mdb/jdbc-details)]
(jdbc/db-set-rollback-only! target-db-conn)
(println (u/format-color 'blue "Temporarily disabling DB constraints..."))
(disable-db-constraints! target-db-conn)
(println-ok)
(load-data! target-db-conn h2-connection-string-or-nil)
(reënable-db-constraints-if-needed! (mdb/jdbc-details))
(println (u/format-color 'blue "Re-enabling DB constraints..."))
(reenable-db-constraints! target-db-conn)
(println-ok)
(jdbc/db-unset-rollback-only! target-db-conn))
(set-postgres-sequence-values-if-needed!))
(update-sequence-values!))
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment