Skip to content
Snippets Groups Projects
Unverified Commit 90151000 authored by Cam Saul's avatar Cam Saul Committed by GitHub
Browse files

Merge pull request #9726 from metabase/add-migrate-from-h2-to-ci 

Add tests for load-from-h2 to CI
parents 36c85785 f4622559
Branches
Tags
No related merge requests found
Hide whitespace changes
Inline Side-by-side
.circleci/config.yml
+ 57
− 0
View file @ 90151000
......@@ -459,6 +459,55 @@ jobs:
lein with-profile +ci test
no_output_timeout: 5m
be-tests-migrate-to-postgres:
working_directory: /home/circleci/metabase/metabase/
docker:
- image: circleci/clojure:lein-2.8.1-node-browsers
- image: circleci/postgres:9.6-alpine
environment:
POSTGRES_USER: circle_test
POSTGRES_DB: circle_test
steps:
- attach_workspace:
at: /home/circleci/
- restore_cache:
<<: *restore-be-deps-cache
- run:
name: Test migrating from H2 -> Postgres
environment:
MB_DB_TYPE: postgres
MB_DB_PORT: 5432
MB_DB_HOST: localhost
MB_DB_DBNAME: circle_test
MB_DB_USER: circle_test
# Test with relative path / no file: prefix
command: >
lein run load-from-h2 ./frontend/test/__runner__/test_db_fixture.db
no_output_timeout: 5m
be-tests-migrate-to-mysql:
working_directory: /home/circleci/metabase/metabase/
docker:
- image: circleci/clojure:lein-2.8.1-node-browsers
- image: circleci/mysql:5.7.23
steps:
- attach_workspace:
at: /home/circleci/
- restore_cache:
<<: *restore-be-deps-cache
- run:
name: Test migrating from H2 -> MySQL
environment:
MB_DB_TYPE: mysql
MB_DB_HOST: localhost
MB_DB_PORT: 3306
MB_DB_DBNAME: circle_test
MB_DB_USER: root
# Test including file: prefix & absolute path
command: >
lein run load-from-h2 file:/home/circleci/metabase/metabase/frontend/test/__runner__/test_db_fixture.db
no_output_timeout: 5m
########################################################################################################################
# FRONTEND #
......@@ -689,6 +738,12 @@ workflows:
- be-tests-snowflake:
requires:
- be-tests
- be-tests-migrate-to-postgres:
requires:
- be-tests
- be-tests-migrate-to-mysql:
requires:
- be-tests
- yaml-linter:
requires:
- checkout
......@@ -743,6 +798,8 @@ workflows:
- be-tests-redshift
- be-tests-vertica
- be-tests-snowflake
- be-tests-migrate-to-postgres
- be-tests-migrate-to-mysql
- yaml-linter
......
frontend/test/__runner__/test_db_fixture.db.h2.db
+ 0
− 0
View file @ 90151000
No preview for this file type
frontend/test/metabase/services/__snapshots__/MetabaseApi.e2e.spec.js.snap
+ 0
− 78
View file @ 90151000
......@@ -439,8 +439,6 @@ Object {
"fk_target_field_id": null,
"has_field_values": "none",
"id": 1,
"max_value": null,
"min_value": null,
"name": "CREATED_AT",
"parent_id": null,
"points_of_interest": null,
......@@ -535,8 +533,6 @@ Object {
"fk_target_field_id": null,
"has_field_values": "none",
"id": 35,
"max_value": null,
"min_value": null,
"name": "DISCOUNT",
"parent_id": null,
"points_of_interest": null,
......@@ -574,8 +570,6 @@ Object {
"fk_target_field_id": null,
"has_field_values": "none",
"id": 2,
"max_value": 17624,
"min_value": 1,
"name": "ID",
"parent_id": null,
"points_of_interest": null,
......@@ -607,8 +601,6 @@ Object {
"fk_target_field_id": 24,
"has_field_values": "none",
"id": 3,
"max_value": 200,
"min_value": 1,
"name": "PRODUCT_ID",
"parent_id": null,
"points_of_interest": null,
......@@ -640,8 +632,6 @@ Object {
"fk_target_field_id": null,
"has_field_values": "none",
"id": 24,
"max_value": 200,
"min_value": 1,
"name": "ID",
"parent_id": null,
"points_of_interest": null,
......@@ -737,8 +727,6 @@ Object {
"fk_target_field_id": null,
"has_field_values": "list",
"id": 36,
"max_value": null,
"min_value": null,
"name": "QUANTITY",
"parent_id": null,
"points_of_interest": null,
......@@ -833,8 +821,6 @@ Object {
"fk_target_field_id": null,
"has_field_values": "none",
"id": 4,
"max_value": 99.37,
"min_value": 12.02,
"name": "SUBTOTAL",
"parent_id": null,
"points_of_interest": null,
......@@ -929,8 +915,6 @@ Object {
"fk_target_field_id": null,
"has_field_values": "none",
"id": 5,
"max_value": 7.45,
"min_value": 0,
"name": "TAX",
"parent_id": null,
"points_of_interest": null,
......@@ -1025,8 +1009,6 @@ Object {
"fk_target_field_id": null,
"has_field_values": "none",
"id": 6,
"max_value": 106.82000000000001,
"min_value": 12.02,
"name": "TOTAL",
"parent_id": null,
"points_of_interest": null,
......@@ -1058,8 +1040,6 @@ Object {
"fk_target_field_id": 13,
"has_field_values": "none",
"id": 7,
"max_value": 2498,
"min_value": 2,
"name": "USER_ID",
"parent_id": null,
"points_of_interest": null,
......@@ -1091,8 +1071,6 @@ Object {
"fk_target_field_id": null,
"has_field_values": "none",
"id": 13,
"max_value": 2500,
"min_value": 1,
"name": "ID",
"parent_id": null,
"points_of_interest": null,
......@@ -1415,8 +1393,6 @@ Object {
"fk_target_field_id": null,
"has_field_values": "search",
"id": 8,
"max_value": null,
"min_value": null,
"name": "ADDRESS",
"parent_id": null,
"points_of_interest": null,
......@@ -1598,8 +1574,6 @@ Object {
"fk_target_field_id": null,
"has_field_values": "none",
"id": 9,
"max_value": null,
"min_value": null,
"name": "BIRTH_DATE",
"parent_id": null,
"points_of_interest": null,
......@@ -1639,8 +1613,6 @@ Object {
"fk_target_field_id": null,
"has_field_values": "search",
"id": 10,
"max_value": null,
"min_value": null,
"name": "CITY",
"parent_id": null,
"points_of_interest": null,
......@@ -1822,8 +1794,6 @@ Object {
"fk_target_field_id": null,
"has_field_values": "none",
"id": 11,
"max_value": null,
"min_value": null,
"name": "CREATED_AT",
"parent_id": null,
"points_of_interest": null,
......@@ -1863,8 +1833,6 @@ Object {
"fk_target_field_id": null,
"has_field_values": "search",
"id": 12,
"max_value": null,
"min_value": null,
"name": "EMAIL",
"parent_id": null,
"points_of_interest": null,
......@@ -1902,8 +1870,6 @@ Object {
"fk_target_field_id": null,
"has_field_values": "none",
"id": 13,
"max_value": 2500,
"min_value": 1,
"name": "ID",
"parent_id": null,
"points_of_interest": null,
......@@ -2008,8 +1974,6 @@ Object {
"fk_target_field_id": null,
"has_field_values": "none",
"id": 14,
"max_value": 89.98241873383432,
"min_value": -89.96310010740648,
"name": "LATITUDE",
"parent_id": null,
"points_of_interest": null,
......@@ -2114,8 +2078,6 @@ Object {
"fk_target_field_id": null,
"has_field_values": "none",
"id": 15,
"max_value": 179.73650520575882,
"min_value": -179.30480334715446,
"name": "LONGITUDE",
"parent_id": null,
"points_of_interest": null,
......@@ -2155,8 +2117,6 @@ Object {
"fk_target_field_id": null,
"has_field_values": "search",
"id": 16,
"max_value": null,
"min_value": null,
"name": "NAME",
"parent_id": null,
"points_of_interest": null,
......@@ -2196,8 +2156,6 @@ Object {
"fk_target_field_id": null,
"has_field_values": "search",
"id": 17,
"max_value": null,
"min_value": null,
"name": "PASSWORD",
"parent_id": null,
"points_of_interest": null,
......@@ -2237,8 +2195,6 @@ Object {
"fk_target_field_id": null,
"has_field_values": "list",
"id": 18,
"max_value": null,
"min_value": null,
"name": "SOURCE",
"parent_id": null,
"points_of_interest": null,
......@@ -2278,8 +2234,6 @@ Object {
"fk_target_field_id": null,
"has_field_values": "list",
"id": 19,
"max_value": null,
"min_value": null,
"name": "STATE",
"parent_id": null,
"points_of_interest": null,
......@@ -2319,8 +2273,6 @@ Object {
"fk_target_field_id": null,
"has_field_values": "search",
"id": 20,
"max_value": null,
"min_value": null,
"name": "ZIP",
"parent_id": null,
"points_of_interest": null,
......@@ -2642,8 +2594,6 @@ Object {
"fk_target_field_id": null,
"has_field_values": "list",
"id": 21,
"max_value": null,
"min_value": null,
"name": "CATEGORY",
"parent_id": null,
"points_of_interest": null,
......@@ -2825,8 +2775,6 @@ Object {
"fk_target_field_id": null,
"has_field_values": "none",
"id": 22,
"max_value": null,
"min_value": null,
"name": "CREATED_AT",
"parent_id": null,
"points_of_interest": null,
......@@ -2866,8 +2814,6 @@ Object {
"fk_target_field_id": null,
"has_field_values": "search",
"id": 23,
"max_value": null,
"min_value": null,
"name": "EAN",
"parent_id": null,
"points_of_interest": null,
......@@ -2905,8 +2851,6 @@ Object {
"fk_target_field_id": null,
"has_field_values": "none",
"id": 24,
"max_value": 200,
"min_value": 1,
"name": "ID",
"parent_id": null,
"points_of_interest": null,
......@@ -3001,8 +2945,6 @@ Object {
"fk_target_field_id": null,
"has_field_values": "none",
"id": 25,
"max_value": 99.37,
"min_value": 12.02,
"name": "PRICE",
"parent_id": null,
"points_of_interest": null,
......@@ -3097,8 +3039,6 @@ Object {
"fk_target_field_id": null,
"has_field_values": "list",
"id": 26,
"max_value": 5,
"min_value": 0,
"name": "RATING",
"parent_id": null,
"points_of_interest": null,
......@@ -3138,8 +3078,6 @@ Object {
"fk_target_field_id": null,
"has_field_values": "search",
"id": 27,
"max_value": null,
"min_value": null,
"name": "TITLE",
"parent_id": null,
"points_of_interest": null,
......@@ -3179,8 +3117,6 @@ Object {
"fk_target_field_id": null,
"has_field_values": "search",
"id": 28,
"max_value": null,
"min_value": null,
"name": "VENDOR",
"parent_id": null,
"points_of_interest": null,
......@@ -3502,8 +3438,6 @@ Object {
"fk_target_field_id": null,
"has_field_values": "search",
"id": 29,
"max_value": null,
"min_value": null,
"name": "BODY",
"parent_id": null,
"points_of_interest": null,
......@@ -3685,8 +3619,6 @@ Object {
"fk_target_field_id": null,
"has_field_values": "none",
"id": 30,
"max_value": null,
"min_value": null,
"name": "CREATED_AT",
"parent_id": null,
"points_of_interest": null,
......@@ -3724,8 +3656,6 @@ Object {
"fk_target_field_id": null,
"has_field_values": "none",
"id": 31,
"max_value": 1078,
"min_value": 1,
"name": "ID",
"parent_id": null,
"points_of_interest": null,
......@@ -3757,8 +3687,6 @@ Object {
"fk_target_field_id": 24,
"has_field_values": "none",
"id": 32,
"max_value": 200,
"min_value": 1,
"name": "PRODUCT_ID",
"parent_id": null,
"points_of_interest": null,
......@@ -3790,8 +3718,6 @@ Object {
"fk_target_field_id": null,
"has_field_values": "none",
"id": 24,
"max_value": 200,
"min_value": 1,
"name": "ID",
"parent_id": null,
"points_of_interest": null,
......@@ -3887,8 +3813,6 @@ Object {
"fk_target_field_id": null,
"has_field_values": "list",
"id": 33,
"max_value": 5,
"min_value": 1,
"name": "RATING",
"parent_id": null,
"points_of_interest": null,
......@@ -3928,8 +3852,6 @@ Object {
"fk_target_field_id": null,
"has_field_values": "search",
"id": 34,
"max_value": null,
"min_value": null,
"name": "REVIEWER",
"parent_id": null,
"points_of_interest": null,
......
src/metabase/cmd/load_from_h2.clj
+ 75
− 59
View file @ 90151000
......@@ -14,14 +14,15 @@
mysql -u root -e 'DROP DATABASE IF EXISTS metabase; CREATE DATABASE metabase;'
MB_DB_TYPE=mysql MB_DB_HOST=localhost MB_DB_PORT=3305 MB_DB_USER=root MB_DB_DBNAME=metabase lein run load-from-h2
```"
(:require [clojure.java
(:require [clojure
[set :as set]
[string :as str]]
[clojure.java
[io :as io]
[jdbc :as jdbc]]
[clojure.string :as str]
[colorize.core :as color]
[medley.core :as m]
[metabase
[config :as config]
[db :as mdb]
[util :as u]]
[metabase.db.migrations :refer [DataMigrations]]
......@@ -56,11 +57,15 @@
[setting :refer [Setting]]
[table :refer [Table]]
[user :refer [User]]
[view-log :refer [ViewLog]]]))
[view-log :refer [ViewLog]]]
[toucan.db :as db])
(:import java.sql.SQLException))
(defn- println-ok [] (println (color/green "[OK]")))
;;; -------------------------------------------------- Loading Data --------------------------------------------------
(defn- dispatch-on-db-type [& _] (mdb/db-type))
;;; ------------------------------------------ Models to Migrate (in order) ------------------------------------------
(def ^:private entities
"Entities in the order they should be serialized/deserialized. This is done so we make sure that we load load
......@@ -100,6 +105,9 @@
;; above this line)
DataMigrations])
;;; --------------------------------------------- H2 Connection Options ----------------------------------------------
(defn- add-file-prefix-if-needed [connection-string-or-filename]
(if (str/starts-with? connection-string-or-filename "file:")
connection-string-or-filename
......@@ -110,31 +118,28 @@
(mdb/jdbc-details {:type :h2, :db (str h2-filename ";IFEXISTS=TRUE")})))
(defn- insert-entity! [target-db-conn entity objs]
(print (u/format-color 'blue "Transfering %d instances of %s..." (count objs) (:name entity))) ; TODO - I don't think the print+flush is working as intended :/
;;; ------------------------------------------- Fetching & Inserting Rows --------------------------------------------
(defn- insert-entity! [target-db-conn {:keys [table], :as entity} objs]
;; TODO - I don't think the print+flush is working as intended :/
(print (u/format-color 'blue "Transfering %d instances of %s..." (count objs) (:name entity)))
(flush)
(let [ks (keys (first objs))
;; 1) `:sizeX` and `:sizeY` come out of H2 as `:sizex` and `:sizey` because of automatic lowercasing; fix the
;; names of these before putting into the new DB
;; 2) Need to wrap the column names in quotes because Postgres automatically lowercases unquoted identifiers
quote-char (case (config/config-kw :mb-db-type)
:postgres \"
:mysql \`)
cols (for [k ks]
(str quote-char (name (case k
:sizex :sizeX
:sizey :sizeY
k)) quote-char))]
;; 1) `:sizeX` and `:sizeY` come out of H2 as `:sizex` and `:sizey` because of automatic lowercasing; fix the
;; names of these before putting into the new DB
;; 2) Need to wrap the column names in quotes because Postgres automatically lowercases unquoted identifiers
(let [ks (keys (set/rename-keys (first objs) {:sizex :sizeX, :sizey :sizeY}))
cols (for [k ks]
((db/quote-fn) (name k)))]
;; The connection closes prematurely on occasion when we're inserting thousands of rows at once. Break into
;; smaller chunks so connection stays alive
(doseq [chunk (partition-all 300 objs)]
(doseq [chunk (partition-all 300 objs)
:let [vals (for [row chunk]
(map row ks))]]
(print (color/blue \.))
(flush)
(try
(jdbc/insert-multi! target-db-conn (:table entity) cols (for [row chunk]
(map row ks)))
(catch java.sql.SQLException e
(jdbc/insert-multi! target-db-conn table cols vals)
(catch SQLException e
(jdbc/print-sql-exception-chain e)
(throw e)))))
(println-ok))
......@@ -151,7 +156,16 @@
;;; ---------------------------------------- Enabling / Disabling Constraints ----------------------------------------
(defn- disable-db-constraints:postgres! [target-db-conn]
(defmulti ^:private disable-db-constraints!
{:arglists '([target-db-conn])}
dispatch-on-db-type)
(defmulti ^:private reenable-db-constraints!
{:arglists '([target-db-conn])}
dispatch-on-db-type)
(defmethod disable-db-constraints! :postgres [target-db-conn]
;; make all of our FK constraints deferrable. This only works on Postgres 9.4+ (December 2014)! (There's no pressing
;; reason to turn these back on at the conclusion of this script. It makes things more complicated since it doesn't
;; work if done inside the same transaction.)
......@@ -164,49 +178,41 @@
;; now enable constraint deferring for the duration of the transaction
(jdbc/execute! target-db-conn ["SET CONSTRAINTS ALL DEFERRED"]))
(defmethod reenable-db-constraints! :postgres [_]) ; no-op
(defn- disable-db-constraints:mysql! [target-db-conn]
(defmethod disable-db-constraints! :mysql [target-db-conn]
(jdbc/execute! target-db-conn ["SET FOREIGN_KEY_CHECKS=0"]))
;; For MySQL we need to reënable FK checks when we're done
(defn- reënable-db-constraints:mysql! [target-db-conn]
;; For MySQL we need to re-enable FK checks when we're done
(defmethod reenable-db-constraints! :mysql [target-db-conn]
(jdbc/execute! target-db-conn ["SET FOREIGN_KEY_CHECKS=1"]))
(defn- disable-db-constraints! [target-db-conn]
(println (u/format-color 'blue "Temporarily disabling DB constraints..."))
((case (mdb/db-type)
:postgres disable-db-constraints:postgres!
:mysql disable-db-constraints:mysql!) target-db-conn)
(println-ok))
(defn- reënable-db-constraints-if-needed! [target-db-conn]
(when (= (mdb/db-type) :mysql)
(println (u/format-color 'blue "Reënabling DB constraints..."))
(reënable-db-constraints:mysql! target-db-conn)
(println-ok)))
;;; ---------------------------------------- Fixing Postgres Sequence Values -----------------------------------------
;;; --------------------------------------------- Fixing Sequence Values ---------------------------------------------
(def ^:private entities-without-autoinc-ids
"Entities that do NOT use an auto incrementing ID column."
#{Setting Session DataMigrations})
(defn- set-postgres-sequence-values-if-needed!
"When loading data into a Postgres DB, update the sequence nextvals."
[]
(when (= (mdb/db-type) :postgres)
(jdbc/with-db-transaction [target-db-conn (mdb/jdbc-details)]
(println (u/format-color 'blue "Setting postgres sequence ids to proper values..."))
(doseq [e entities
:when (not (contains? entities-without-autoinc-ids e))
:let [table-name (name (:table e))
seq-name (str table-name "_id_seq")
sql (format "SELECT setval('%s', COALESCE((SELECT MAX(id) FROM %s), 1), true) as val"
seq-name (name table-name))]]
(jdbc/db-query-with-resultset target-db-conn [sql] :val))
(println-ok))))
(defmulti ^:private update-sequence-values!
{:arglists '([])}
dispatch-on-db-type)
(defmethod update-sequence-values! :mysql []) ; no-op
;; Update the sequence nextvals.
(defmethod update-sequence-values! :postgres []
(jdbc/with-db-transaction [target-db-conn (mdb/jdbc-details)]
(println (u/format-color 'blue "Setting postgres sequence ids to proper values..."))
(doseq [e entities
:when (not (contains? entities-without-autoinc-ids e))
:let [table-name (name (:table e))
seq-name (str table-name "_id_seq")
sql (format "SELECT setval('%s', COALESCE((SELECT MAX(id) FROM %s), 1), true) as val"
seq-name (name table-name))]]
(jdbc/db-query-with-resultset target-db-conn [sql] :val))
(println-ok)))
;;; --------------------------------------------------- Public Fns ---------------------------------------------------
......@@ -218,10 +224,20 @@
Defaults to using `@metabase.db/db-file` as the connection string."
[h2-connection-string-or-nil]
(mdb/setup-db!)
(jdbc/with-db-transaction [target-db-conn (mdb/jdbc-details)]
(jdbc/db-set-rollback-only! target-db-conn)
(println (u/format-color 'blue "Temporarily disabling DB constraints..."))
(disable-db-constraints! target-db-conn)
(println-ok)
(load-data! target-db-conn h2-connection-string-or-nil)
(reënable-db-constraints-if-needed! (mdb/jdbc-details))
(println (u/format-color 'blue "Re-enabling DB constraints..."))
(reenable-db-constraints! target-db-conn)
(println-ok)
(jdbc/db-unset-rollback-only! target-db-conn))
(set-postgres-sequence-values-if-needed!))
(update-sequence-values!))
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment