Merge pull request #2712 from metabase/load-from-h2-fixes

Fixes for load-from-h2 migration command

resources/log4j.properties

@@ -18,7 +18,6 @@ log4j.appender.metabase=metabase.logger.Appender
 
 # customizations to logging by package
 log4j.logger.metabase=INFO
-log4j.logger.metabase.db=DEBUG
 log4j.logger.metabase.driver=DEBUG
 log4j.logger.metabase.middleware=DEBUG
 log4j.logger.metabase.query-processor=DEBUG

src/metabase/cmd/load_from_h2.clj

 (ns metabase.cmd.load-from-h2
   "Commands for loading data from an H2 file into another database."
-  (:require [colorize.core :as color]
+  (:require [clojure.java.jdbc :as jdbc]
+            [clojure.set :as set]
+            [colorize.core :as color]
             (korma [core :as k]
                    [db :as kdb])
+            [medley.core :as m]
+            [metabase.config :as config]
             [metabase.db :as db]
             (metabase.models [activity :refer [Activity]]
                              [card :refer [Card]]
@@ -67,27 +71,77 @@
    Label
    CardLabel])
 
+(def ^:private self-referencing-entities
+  "Entities that have a column with and FK that points back to the same table."
+  #{RawColumn Field})
+
+(def ^:private entities-without-autoinc-ids
+  "Entities that do NOT use an auto incrementing ID column."
+  #{Setting Session})
+
+(def ^:private ^:dynamic *db-conn*
+  "Active database connection to the target database we are loading into."
+  nil)
+
+(defn- insert-entity [e objs]
+  (print (u/format-color 'blue "Transfering %d instances of %s..." (count objs) (:name e)))
+  (flush)
+  ;; The connection closes prematurely on occasion when we're inserting thousands of rows at once. Break into smaller chunks so connection stays alive
+  (doseq [chunk (partition-all 300 objs)]
+    (print (color/blue \.))
+    (flush)
+    (k/insert e (k/values (if (= e DashboardCard)
+                            ;; mini-HACK to fix korma/h2 lowercasing these couple attributes
+                            ;; luckily this is the only place in our schema where we have camel case names
+                            (mapv #(set/rename-keys % {:sizex :sizeX, :sizey :sizeY}) chunk)
+                            chunk))))
+  (println (color/green "[OK]")))
+
+(defn- insert-self-referencing-entity [e objs]
+  (let [self-ref-attr    (condp = e
+                           RawColumn :fk_target_column_id
+                           Field     :fk_target_field_id)
+        self-referencing (filter self-ref-attr objs)
+        others           (set/difference (set objs) (set self-referencing))]
+    ;; first insert the non-self-referencing objects
+    (insert-entity e others)
+    ;; then insert the rest, which should be safe to insert now
+    (insert-entity e self-referencing)))
+
+(defn- set-postgres-sequence-values []
+  (print (u/format-color 'blue "Setting postgres sequence ids to proper values..."))
+  (flush)
+  (doseq [e    (filter #(not (contains? entities-without-autoinc-ids %)) entities)
+          :let [table-name (:table e)
+                seq-name   (str table-name "_id_seq")
+                sql        (format "SELECT setval('%s', COALESCE((SELECT MAX(id) FROM %s), 1), true) as val" seq-name table-name)]]
+    (jdbc/db-query-with-resultset *db-conn* [sql] :val))
+  (println (color/green "[OK]")))
+
 (defn load-from-h2
   "Transfer data from existing H2 database to the newly created (presumably MySQL or Postgres) DB specified by env vars.
    Intended as a tool for upgrading from H2 to a 'real' Database.
 
    Defaults to using `@metabase.db/db-file` as the connection string."
   [h2-connection-string-or-nil]
-  (let [filename (or h2-connection-string-or-nil
-                     @metabase.db/db-file)
-        h2-db    (kdb/create-db (db/jdbc-details {:type :h2, :db (str filename ";IFEXISTS=TRUE")}))] ; TODO - would be nice to add `ACCESS_MODE_DATA=r` but it doesn't work with `AUTO_SERVER=TRUE`
-    (db/setup-db)
-    (kdb/transaction
-     (doseq [e     entities
-             :let  [objs (kdb/with-db h2-db
-                           (k/select (k/database e h2-db)))]
-             :when (seq objs)]
-       (print (u/format-color 'blue "Transfering %d instances of %s..." (count objs) (:name e)))
-       (flush)
+  (db/setup-db)
+  (let [h2-filename (or h2-connection-string-or-nil @metabase.db/db-file)]
+    ;; NOTE: would be nice to add `ACCESS_MODE_DATA=r` but it doesn't work with `AUTO_SERVER=TRUE`
+    ;; connect to H2 database, which is what we are migrating from
+    (jdbc/with-db-connection [h2-conn (db/jdbc-details {:type :h2, :db (str h2-filename ";IFEXISTS=TRUE")})]
+      (kdb/transaction
+        (doseq [e     entities
+                :let  [objs (->> (jdbc/query h2-conn [(str "SELECT * FROM " (:table e))])
+                                 ;; we apply jdbc-clob->str to all row values because H2->Postgres
+                                 ;; gets messed up if the value is left as a clob
+                                 (map #(m/map-vals u/jdbc-clob->str %)))]
+                :when (seq objs)]
+          (if-not (contains? self-referencing-entities e)
+            (insert-entity e objs)
+            (insert-self-referencing-entity e objs)))))
 
-       ;; The connection closes prematurely on occasion when we're inserting thousands of rows at once. Break into smaller chunks so connection stays alive
-       (doseq [chunk (partition-all 300 objs)]
-         (print (color/blue \.))
-         (flush)
-         (k/insert e (k/values chunk)))
-       (println (color/green "[OK]"))))))
+    ;; if we are loading into a postgres db then we need to update sequence nextvals
+    (when (= (config/config-str :mb-db-type) "postgres")
+      (jdbc/with-db-transaction [targetdb-conn (db/jdbc-details @db/db-connection-details)]
+        (binding [*db-conn* targetdb-conn]
+          (set-postgres-sequence-values))))))