From f77c473716e5ef9b1c921c5232a8d2d077b2d126 Mon Sep 17 00:00:00 2001
From: Cam Saul <1455846+camsaul@users.noreply.github.com>
Date: Tue, 4 Feb 2020 11:21:54 -0800
Subject: [PATCH] Automatically convert MySQL/MariaDB application DBs to
utf8mb4 on launch (#11753)
---
resources/migrations/000_migrations.yaml | 872 ++++++++++++++++++
src/metabase/db.clj | 14 +-
test/metabase/db/fix_mysql_utf8_test.clj | 111 +++
.../date_bucketing_test.clj | 8 +-
4 files changed, 994 insertions(+), 11 deletions(-)
create mode 100644 test/metabase/db/fix_mysql_utf8_test.clj
diff --git a/resources/migrations/000_migrations.yaml b/resources/migrations/000_migrations.yaml
index 141a14774f8..6eb884981d6 100644
--- a/resources/migrations/000_migrations.yaml
+++ b/resources/migrations/000_migrations.yaml
@@ -5304,3 +5304,875 @@ databaseChangeLog:
tableName: metabase_field
columnName: database_type
newDataType: text
+
+#
+# Migrations 107-160 are used to convert a MySQL or MariaDB database to utf8mb4 on launch -- see #11753 for a detailed explanation of these migrations
+#
+
+ - changeSet:
+ id: 107
+ author: camsaul
+ comment: Added 0.34.2
+ # If this migration fails for any reason continue with the next migration; do not fail the entire process if this one fails
+ failOnError: false
+ preConditions:
+ # If preconditions fail (i.e., dbms is not mysql or mariadb) then mark this migration as 'ran'
+ - onFail: MARK_RAN
+ # If we're generating SQL output for migrations instead of running via liquibase, fail the preconditions which means these migrations will be skipped
+ - onSqlOutput: FAIL
+ - or:
+ - dbms:
+ type: mysql
+ - dbms:
+ type: mariadb
+ changes:
+ - sql:
+ sql: ALTER DATABASE CHARACTER SET = utf8mb4 COLLATE = utf8mb4_unicode_ci;
+ - changeSet:
+ id: 108
+ author: camsaul
+ comment: Added 0.34.2
+ failOnError: false
+ preConditions:
+ - onFail: MARK_RAN
+ - onSqlOutput: FAIL
+ - or:
+ - dbms:
+ type: mysql
+ - dbms:
+ type: mariadb
+ changes:
+ - sql:
+ sql: ALTER TABLE `DATABASECHANGELOG` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+ - changeSet:
+ id: 109
+ author: camsaul
+ comment: Added 0.34.2
+ failOnError: false
+ preConditions:
+ - onFail: MARK_RAN
+ - onSqlOutput: FAIL
+ - or:
+ - dbms:
+ type: mysql
+ - dbms:
+ type: mariadb
+ changes:
+ - sql:
+ sql: ALTER TABLE `DATABASECHANGELOGLOCK` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+ - changeSet:
+ id: 110
+ author: camsaul
+ comment: Added 0.34.2
+ failOnError: false
+ preConditions:
+ - onFail: MARK_RAN
+ - onSqlOutput: FAIL
+ - or:
+ - dbms:
+ type: mysql
+ - dbms:
+ type: mariadb
+ changes:
+ - sql:
+ sql: ALTER TABLE `QRTZ_CALENDARS` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+ - changeSet:
+ id: 111
+ author: camsaul
+ comment: Added 0.34.2
+ failOnError: false
+ preConditions:
+ - onFail: MARK_RAN
+ - onSqlOutput: FAIL
+ - or:
+ - dbms:
+ type: mysql
+ - dbms:
+ type: mariadb
+ changes:
+ - sql:
+ sql: ALTER TABLE `QRTZ_FIRED_TRIGGERS` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+ - changeSet:
+ id: 112
+ author: camsaul
+ comment: Added 0.34.2
+ failOnError: false
+ preConditions:
+ - onFail: MARK_RAN
+ - onSqlOutput: FAIL
+ - or:
+ - dbms:
+ type: mysql
+ - dbms:
+ type: mariadb
+ changes:
+ - sql:
+ sql: ALTER TABLE `QRTZ_JOB_DETAILS` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+ - changeSet:
+ id: 113
+ author: camsaul
+ comment: Added 0.34.2
+ failOnError: false
+ preConditions:
+ - onFail: MARK_RAN
+ - onSqlOutput: FAIL
+ - or:
+ - dbms:
+ type: mysql
+ - dbms:
+ type: mariadb
+ changes:
+ - sql:
+ sql: ALTER TABLE `QRTZ_LOCKS` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+ - changeSet:
+ id: 114
+ author: camsaul
+ comment: Added 0.34.2
+ failOnError: false
+ preConditions:
+ - onFail: MARK_RAN
+ - onSqlOutput: FAIL
+ - or:
+ - dbms:
+ type: mysql
+ - dbms:
+ type: mariadb
+ changes:
+ - sql:
+ sql: ALTER TABLE `QRTZ_PAUSED_TRIGGER_GRPS` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+ - changeSet:
+ id: 115
+ author: camsaul
+ comment: Added 0.34.2
+ failOnError: false
+ preConditions:
+ - onFail: MARK_RAN
+ - onSqlOutput: FAIL
+ - or:
+ - dbms:
+ type: mysql
+ - dbms:
+ type: mariadb
+ changes:
+ - sql:
+ sql: ALTER TABLE `QRTZ_SCHEDULER_STATE` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+ - changeSet:
+ id: 116
+ author: camsaul
+ comment: Added 0.34.2
+ failOnError: false
+ preConditions:
+ - onFail: MARK_RAN
+ - onSqlOutput: FAIL
+ - or:
+ - dbms:
+ type: mysql
+ - dbms:
+ type: mariadb
+ changes:
+ - sql:
+ sql: ALTER TABLE `core_user` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+ - changeSet:
+ id: 117
+ author: camsaul
+ comment: Added 0.34.2
+ failOnError: false
+ preConditions:
+ - onFail: MARK_RAN
+ - onSqlOutput: FAIL
+ - or:
+ - dbms:
+ type: mysql
+ - dbms:
+ type: mariadb
+ changes:
+ - sql:
+ sql: ALTER TABLE `data_migrations` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+ - changeSet:
+ id: 118
+ author: camsaul
+ comment: Added 0.34.2
+ failOnError: false
+ preConditions:
+ - onFail: MARK_RAN
+ - onSqlOutput: FAIL
+ - or:
+ - dbms:
+ type: mysql
+ - dbms:
+ type: mariadb
+ changes:
+ - sql:
+ sql: ALTER TABLE `dependency` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+ - changeSet:
+ id: 119
+ author: camsaul
+ comment: Added 0.34.2
+ failOnError: false
+ preConditions:
+ - onFail: MARK_RAN
+ - onSqlOutput: FAIL
+ - or:
+ - dbms:
+ type: mysql
+ - dbms:
+ type: mariadb
+ changes:
+ - sql:
+ sql: ALTER TABLE `label` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+ - changeSet:
+ id: 120
+ author: camsaul
+ comment: Added 0.34.2
+ failOnError: false
+ preConditions:
+ - onFail: MARK_RAN
+ - onSqlOutput: FAIL
+ - or:
+ - dbms:
+ type: mysql
+ - dbms:
+ type: mariadb
+ changes:
+ - sql:
+ sql: ALTER TABLE `metabase_database` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+ - changeSet:
+ id: 121
+ author: camsaul
+ comment: Added 0.34.2
+ failOnError: false
+ preConditions:
+ - onFail: MARK_RAN
+ - onSqlOutput: FAIL
+ - or:
+ - dbms:
+ type: mysql
+ - dbms:
+ type: mariadb
+ changes:
+ - sql:
+ sql: ALTER TABLE `permissions_group` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+ - changeSet:
+ id: 122
+ author: camsaul
+ comment: Added 0.34.2
+ failOnError: false
+ preConditions:
+ - onFail: MARK_RAN
+ - onSqlOutput: FAIL
+ - or:
+ - dbms:
+ type: mysql
+ - dbms:
+ type: mariadb
+ changes:
+ - sql:
+ sql: ALTER TABLE `query` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+ - changeSet:
+ id: 123
+ author: camsaul
+ comment: Added 0.34.2
+ failOnError: false
+ preConditions:
+ - onFail: MARK_RAN
+ - onSqlOutput: FAIL
+ - or:
+ - dbms:
+ type: mysql
+ - dbms:
+ type: mariadb
+ changes:
+ - sql:
+ sql: ALTER TABLE `query_cache` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+ - changeSet:
+ id: 124
+ author: camsaul
+ comment: Added 0.34.2
+ failOnError: false
+ preConditions:
+ - onFail: MARK_RAN
+ - onSqlOutput: FAIL
+ - or:
+ - dbms:
+ type: mysql
+ - dbms:
+ type: mariadb
+ changes:
+ - sql:
+ sql: ALTER TABLE `query_execution` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+ - changeSet:
+ id: 125
+ author: camsaul
+ comment: Added 0.34.2
+ failOnError: false
+ preConditions:
+ - onFail: MARK_RAN
+ - onSqlOutput: FAIL
+ - or:
+ - dbms:
+ type: mysql
+ - dbms:
+ type: mariadb
+ changes:
+ - sql:
+ sql: ALTER TABLE `setting` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+ - changeSet:
+ id: 126
+ author: camsaul
+ comment: Added 0.34.2
+ failOnError: false
+ preConditions:
+ - onFail: MARK_RAN
+ - onSqlOutput: FAIL
+ - or:
+ - dbms:
+ type: mysql
+ - dbms:
+ type: mariadb
+ changes:
+ - sql:
+ sql: ALTER TABLE `task_history` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+ - changeSet:
+ id: 127
+ author: camsaul
+ comment: Added 0.34.2
+ failOnError: false
+ preConditions:
+ - onFail: MARK_RAN
+ - onSqlOutput: FAIL
+ - or:
+ - dbms:
+ type: mysql
+ - dbms:
+ type: mariadb
+ changes:
+ - sql:
+ sql: ALTER TABLE `QRTZ_TRIGGERS` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+ - changeSet:
+ id: 128
+ author: camsaul
+ comment: Added 0.34.2
+ failOnError: false
+ preConditions:
+ - onFail: MARK_RAN
+ - onSqlOutput: FAIL
+ - or:
+ - dbms:
+ type: mysql
+ - dbms:
+ type: mariadb
+ changes:
+ - sql:
+ sql: ALTER TABLE `activity` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+ - changeSet:
+ id: 129
+ author: camsaul
+ comment: Added 0.34.2
+ failOnError: false
+ preConditions:
+ - onFail: MARK_RAN
+ - onSqlOutput: FAIL
+ - or:
+ - dbms:
+ type: mysql
+ - dbms:
+ type: mariadb
+ changes:
+ - sql:
+ sql: ALTER TABLE `collection` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+ - changeSet:
+ id: 130
+ author: camsaul
+ comment: Added 0.34.2
+ failOnError: false
+ preConditions:
+ - onFail: MARK_RAN
+ - onSqlOutput: FAIL
+ - or:
+ - dbms:
+ type: mysql
+ - dbms:
+ type: mariadb
+ changes:
+ - sql:
+ sql: ALTER TABLE `collection_revision` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+ - changeSet:
+ id: 131
+ author: camsaul
+ comment: Added 0.34.2
+ failOnError: false
+ preConditions:
+ - onFail: MARK_RAN
+ - onSqlOutput: FAIL
+ - or:
+ - dbms:
+ type: mysql
+ - dbms:
+ type: mariadb
+ changes:
+ - sql:
+ sql: ALTER TABLE `computation_job` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+ - changeSet:
+ id: 132
+ author: camsaul
+ comment: Added 0.34.2
+ failOnError: false
+ preConditions:
+ - onFail: MARK_RAN
+ - onSqlOutput: FAIL
+ - or:
+ - dbms:
+ type: mysql
+ - dbms:
+ type: mariadb
+ changes:
+ - sql:
+ sql: ALTER TABLE `core_session` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+ - changeSet:
+ id: 133
+ author: camsaul
+ comment: Added 0.34.2
+ failOnError: false
+ preConditions:
+ - onFail: MARK_RAN
+ - onSqlOutput: FAIL
+ - or:
+ - dbms:
+ type: mysql
+ - dbms:
+ type: mariadb
+ changes:
+ - sql:
+ sql: ALTER TABLE `metabase_table` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+ - changeSet:
+ id: 134
+ author: camsaul
+ comment: Added 0.34.2
+ failOnError: false
+ preConditions:
+ - onFail: MARK_RAN
+ - onSqlOutput: FAIL
+ - or:
+ - dbms:
+ type: mysql
+ - dbms:
+ type: mariadb
+ changes:
+ - sql:
+ sql: ALTER TABLE `permissions` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+ - changeSet:
+ id: 135
+ author: camsaul
+ comment: Added 0.34.2
+ failOnError: false
+ preConditions:
+ - onFail: MARK_RAN
+ - onSqlOutput: FAIL
+ - or:
+ - dbms:
+ type: mysql
+ - dbms:
+ type: mariadb
+ changes:
+ - sql:
+ sql: ALTER TABLE `permissions_revision` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+ - changeSet:
+ id: 136
+ author: camsaul
+ comment: Added 0.34.2
+ failOnError: false
+ preConditions:
+ - onFail: MARK_RAN
+ - onSqlOutput: FAIL
+ - or:
+ - dbms:
+ type: mysql
+ - dbms:
+ type: mariadb
+ changes:
+ - sql:
+ sql: ALTER TABLE `revision` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+ - changeSet:
+ id: 137
+ author: camsaul
+ comment: Added 0.34.2
+ failOnError: false
+ preConditions:
+ - onFail: MARK_RAN
+ - onSqlOutput: FAIL
+ - or:
+ - dbms:
+ type: mysql
+ - dbms:
+ type: mariadb
+ changes:
+ - sql:
+ sql: ALTER TABLE `view_log` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+ - changeSet:
+ id: 138
+ author: camsaul
+ comment: Added 0.34.2
+ failOnError: false
+ preConditions:
+ - onFail: MARK_RAN
+ - onSqlOutput: FAIL
+ - or:
+ - dbms:
+ type: mysql
+ - dbms:
+ type: mariadb
+ changes:
+ - sql:
+ sql: ALTER TABLE `QRTZ_BLOB_TRIGGERS` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+ - changeSet:
+ id: 139
+ author: camsaul
+ comment: Added 0.34.2
+ failOnError: false
+ preConditions:
+ - onFail: MARK_RAN
+ - onSqlOutput: FAIL
+ - or:
+ - dbms:
+ type: mysql
+ - dbms:
+ type: mariadb
+ changes:
+ - sql:
+ sql: ALTER TABLE `QRTZ_CRON_TRIGGERS` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+ - changeSet:
+ id: 140
+ author: camsaul
+ comment: Added 0.34.2
+ failOnError: false
+ preConditions:
+ - onFail: MARK_RAN
+ - onSqlOutput: FAIL
+ - or:
+ - dbms:
+ type: mysql
+ - dbms:
+ type: mariadb
+ changes:
+ - sql:
+ sql: ALTER TABLE `QRTZ_SIMPLE_TRIGGERS` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+ - changeSet:
+ id: 141
+ author: camsaul
+ comment: Added 0.34.2
+ failOnError: false
+ preConditions:
+ - onFail: MARK_RAN
+ - onSqlOutput: FAIL
+ - or:
+ - dbms:
+ type: mysql
+ - dbms:
+ type: mariadb
+ changes:
+ - sql:
+ sql: ALTER TABLE `QRTZ_SIMPROP_TRIGGERS` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+ - changeSet:
+ id: 142
+ author: camsaul
+ comment: Added 0.34.2
+ failOnError: false
+ preConditions:
+ - onFail: MARK_RAN
+ - onSqlOutput: FAIL
+ - or:
+ - dbms:
+ type: mysql
+ - dbms:
+ type: mariadb
+ changes:
+ - sql:
+ sql: ALTER TABLE `computation_job_result` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+ - changeSet:
+ id: 143
+ author: camsaul
+ comment: Added 0.34.2
+ failOnError: false
+ preConditions:
+ - onFail: MARK_RAN
+ - onSqlOutput: FAIL
+ - or:
+ - dbms:
+ type: mysql
+ - dbms:
+ type: mariadb
+ changes:
+ - sql:
+ sql: ALTER TABLE `metabase_field` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+ - changeSet:
+ id: 144
+ author: camsaul
+ comment: Added 0.34.2
+ failOnError: false
+ preConditions:
+ - onFail: MARK_RAN
+ - onSqlOutput: FAIL
+ - or:
+ - dbms:
+ type: mysql
+ - dbms:
+ type: mariadb
+ changes:
+ - sql:
+ sql: ALTER TABLE `permissions_group_membership` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+ - changeSet:
+ id: 145
+ author: camsaul
+ comment: Added 0.34.2
+ failOnError: false
+ preConditions:
+ - onFail: MARK_RAN
+ - onSqlOutput: FAIL
+ - or:
+ - dbms:
+ type: mysql
+ - dbms:
+ type: mariadb
+ changes:
+ - sql:
+ sql: ALTER TABLE `pulse` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+ - changeSet:
+ id: 146
+ author: camsaul
+ comment: Added 0.34.2
+ failOnError: false
+ preConditions:
+ - onFail: MARK_RAN
+ - onSqlOutput: FAIL
+ - or:
+ - dbms:
+ type: mysql
+ - dbms:
+ type: mariadb
+ changes:
+ - sql:
+ sql: ALTER TABLE `report_dashboard` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+ - changeSet:
+ id: 147
+ author: camsaul
+ comment: Added 0.34.2
+ failOnError: false
+ preConditions:
+ - onFail: MARK_RAN
+ - onSqlOutput: FAIL
+ - or:
+ - dbms:
+ type: mysql
+ - dbms:
+ type: mariadb
+ changes:
+ - sql:
+ sql: ALTER TABLE `dashboard_favorite` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+ - changeSet:
+ id: 148
+ author: camsaul
+ comment: Added 0.34.2
+ failOnError: false
+ preConditions:
+ - onFail: MARK_RAN
+ - onSqlOutput: FAIL
+ - or:
+ - dbms:
+ type: mysql
+ - dbms:
+ type: mariadb
+ changes:
+ - sql:
+ sql: ALTER TABLE `dimension` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+ - changeSet:
+ id: 149
+ author: camsaul
+ comment: Added 0.34.2
+ failOnError: false
+ preConditions:
+ - onFail: MARK_RAN
+ - onSqlOutput: FAIL
+ - or:
+ - dbms:
+ type: mysql
+ - dbms:
+ type: mariadb
+ changes:
+ - sql:
+ sql: ALTER TABLE `metabase_fieldvalues` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+ - changeSet:
+ id: 150
+ author: camsaul
+ comment: Added 0.34.2
+ failOnError: false
+ preConditions:
+ - onFail: MARK_RAN
+ - onSqlOutput: FAIL
+ - or:
+ - dbms:
+ type: mysql
+ - dbms:
+ type: mariadb
+ changes:
+ - sql:
+ sql: ALTER TABLE `metric` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+ - changeSet:
+ id: 151
+ author: camsaul
+ comment: Added 0.34.2
+ failOnError: false
+ preConditions:
+ - onFail: MARK_RAN
+ - onSqlOutput: FAIL
+ - or:
+ - dbms:
+ type: mysql
+ - dbms:
+ type: mariadb
+ changes:
+ - sql:
+ sql: ALTER TABLE `pulse_channel` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+ - changeSet:
+ id: 152
+ author: camsaul
+ comment: Added 0.34.2
+ failOnError: false
+ preConditions:
+ - onFail: MARK_RAN
+ - onSqlOutput: FAIL
+ - or:
+ - dbms:
+ type: mysql
+ - dbms:
+ type: mariadb
+ changes:
+ - sql:
+ sql: ALTER TABLE `segment` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+ - changeSet:
+ id: 153
+ author: camsaul
+ comment: Added 0.34.2
+ failOnError: false
+ preConditions:
+ - onFail: MARK_RAN
+ - onSqlOutput: FAIL
+ - or:
+ - dbms:
+ type: mysql
+ - dbms:
+ type: mariadb
+ changes:
+ - sql:
+ sql: ALTER TABLE `pulse_channel_recipient` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+ - changeSet:
+ id: 154
+ author: camsaul
+ comment: Added 0.34.2
+ failOnError: false
+ preConditions:
+ - onFail: MARK_RAN
+ - onSqlOutput: FAIL
+ - or:
+ - dbms:
+ type: mysql
+ - dbms:
+ type: mariadb
+ changes:
+ - sql:
+ sql: ALTER TABLE `report_card` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+ - changeSet:
+ id: 155
+ author: camsaul
+ comment: Added 0.34.2
+ failOnError: false
+ preConditions:
+ - onFail: MARK_RAN
+ - onSqlOutput: FAIL
+ - or:
+ - dbms:
+ type: mysql
+ - dbms:
+ type: mariadb
+ changes:
+ - sql:
+ sql: ALTER TABLE `metric_important_field` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+ - changeSet:
+ id: 156
+ author: camsaul
+ comment: Added 0.34.2
+ failOnError: false
+ preConditions:
+ - onFail: MARK_RAN
+ - onSqlOutput: FAIL
+ - or:
+ - dbms:
+ type: mysql
+ - dbms:
+ type: mariadb
+ changes:
+ - sql:
+ sql: ALTER TABLE `report_cardfavorite` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+ - changeSet:
+ id: 157
+ author: camsaul
+ comment: Added 0.34.2
+ failOnError: false
+ preConditions:
+ - onFail: MARK_RAN
+ - onSqlOutput: FAIL
+ - or:
+ - dbms:
+ type: mysql
+ - dbms:
+ type: mariadb
+ changes:
+ - sql:
+ sql: ALTER TABLE `card_label` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+ - changeSet:
+ id: 158
+ author: camsaul
+ comment: Added 0.34.2
+ failOnError: false
+ preConditions:
+ - onFail: MARK_RAN
+ - onSqlOutput: FAIL
+ - or:
+ - dbms:
+ type: mysql
+ - dbms:
+ type: mariadb
+ changes:
+ - sql:
+ sql: ALTER TABLE `pulse_card` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+ - changeSet:
+ id: 159
+ author: camsaul
+ comment: Added 0.34.2
+ failOnError: false
+ preConditions:
+ - onFail: MARK_RAN
+ - onSqlOutput: FAIL
+ - or:
+ - dbms:
+ type: mysql
+ - dbms:
+ type: mariadb
+ changes:
+ - sql:
+ sql: ALTER TABLE `report_dashboardcard` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+ - changeSet:
+ id: 160
+ author: camsaul
+ comment: Added 0.34.2
+ failOnError: false
+ preConditions:
+ - onFail: MARK_RAN
+ - onSqlOutput: FAIL
+ - or:
+ - dbms:
+ type: mysql
+ - dbms:
+ type: mariadb
+ changes:
+ - sql:
+ sql: ALTER TABLE `dashboardcard_series` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
diff --git a/src/metabase/db.clj b/src/metabase/db.clj
index 2c17eeb813d..18d96a07918 100644
--- a/src/metabase/db.clj
+++ b/src/metabase/db.clj
@@ -170,10 +170,10 @@
(migrate! :up))
([direction]
- (migrate! @db-connection-details direction))
+ (migrate! (jdbc-spec) direction))
- ([db-details direction]
- (jdbc/with-db-transaction [conn (jdbc-spec db-details)]
+ ([jdbc-spec direction]
+ (jdbc/with-db-transaction [conn jdbc-spec]
;; Tell transaction to automatically `.rollback` instead of `.commit` when the transaction finishes
(jdbc/db-set-rollback-only! conn)
;; Disable auto-commit. This should already be off but set it just to be safe
@@ -270,8 +270,8 @@
([driver :- s/Keyword, details :- su/Map]
(log/info (u/format-color 'cyan (trs "Verifying {0} Database Connection ..." (name driver))))
+ (classloader/require 'metabase.driver.util)
(assert (binding [*allow-potentailly-unsafe-connections* true]
- (classloader/require 'metabase.driver.util)
((resolve 'metabase.driver.util/can-connect-with-details?) driver details :throw-exceptions))
(trs "Unable to connect to Metabase {0} DB." (name driver)))
(jdbc/with-db-metadata [metadata (jdbc-spec details)]
@@ -290,7 +290,7 @@
"If we are not doing auto migrations then print out migration SQL for user to run manually.
Then throw an exception to short circuit the setup process and make it clear we can't proceed."
[db-details]
- (let [sql (migrate! db-details :print)]
+ (let [sql (migrate! (jdbc-spec db-details) :print)]
(log/info (str "Database Upgrade Required\n\n"
"NOTICE: Your database requires updates to work with this version of Metabase. "
"Please execute the following sql commands on your database before proceeding.\n\n"
@@ -304,10 +304,10 @@
[auto-migrate? db-details]
(log/info (trs "Running Database Migrations..."))
(if auto-migrate?
- (migrate! db-details :up)
+ (migrate! (jdbc-spec db-details) :up)
;; if `MB_DB_AUTOMIGRATE` is false, and we have migrations that need to be ran, print and quit. Otherwise continue
;; to start normally
- (when (liquibase/with-liquibase [liquibase (jdbc-spec)]
+ (when (liquibase/with-liquibase [liquibase (jdbc-spec db-details)]
(liquibase/has-unrun-migrations? liquibase))
(print-migrations-and-quit! db-details)))
(log/info (trs "Database Migrations Current ... ") (u/emoji "✅")))
diff --git a/test/metabase/db/fix_mysql_utf8_test.clj b/test/metabase/db/fix_mysql_utf8_test.clj
new file mode 100644
index 00000000000..a6cfc5d92cd
--- /dev/null
+++ b/test/metabase/db/fix_mysql_utf8_test.clj
@@ -0,0 +1,111 @@
+(ns metabase.db.fix-mysql-utf8-test
+ (:require [clojure
+ [string :as str]
+ [test :refer :all]]
+ [clojure.java.jdbc :as jdbc]
+ [metabase
+ [db :as mdb]
+ [models :refer [Database]]
+ [test :as mt]]
+ [metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]
+ [toucan.db :as db]))
+
+(defn- create-test-db! []
+ (jdbc/with-db-connection [server-conn (sql-jdbc.conn/connection-details->spec :mysql
+ (mt/dbdef->connection-details :mysql :server nil))]
+ (doseq [statement ["DROP DATABASE IF EXISTS utf8_test;"
+ "CREATE DATABASE utf8_test;"]]
+ (jdbc/execute! server-conn statement))))
+
+(defn- test-db-spec []
+ (sql-jdbc.conn/connection-details->spec :mysql
+ (mt/dbdef->connection-details :mysql :db {:database-name "utf8_test"})))
+
+(defn- convert-to-charset!
+ "Convert a MySQL/MariaDB database to the `latin1` character set."
+ [jdbc-spec charset collation]
+ (jdbc/with-db-connection [conn jdbc-spec]
+ (doseq [statement [(format "ALTER DATABASE utf8_test CHARACTER SET = %s COLLATE = %s;" charset collation)
+ (format "ALTER TABLE metabase_database CONVERT TO CHARACTER SET %s COLLATE %s;" charset collation)]]
+ (jdbc/execute! jdbc-spec [statement]))))
+
+(defn- remove-utf8mb4-migrations!
+ "Remove the entries for the migrations that convert a DB to utf8mb4 from the Liquibase migration log so they can be
+ ran again."
+ [jdbc-spec]
+ (jdbc/execute! jdbc-spec [(format "DELETE FROM `DATABASECHANGELOG` WHERE ID IN (%s);"
+ (str/join "," (map #(str \' % \')
+ (range 107 161))))]))
+
+(defn- db-charset []
+ (first (jdbc/query db/*db-connection*
+ (str "SELECT default_collation_name AS `collation`, default_character_set_name AS `character-set` "
+ "FROM information_schema.SCHEMATA "
+ "WHERE schema_name = 'utf8_test';"))))
+
+(defn- table-charset []
+ (first (jdbc/query db/*db-connection*
+ (str "SELECT ccsa.collation_name AS `collation`, ccsa.character_set_name AS `character-set` "
+ "FROM information_schema.`TABLES` t,"
+ " information_schema.`COLLATION_CHARACTER_SET_APPLICABILITY` ccsa "
+ "WHERE ccsa.collation_name = t.table_collation"
+ " AND t.table_schema = 'utf8_test' "
+ " AND t.table_name = 'metabase_database';"))))
+
+(defn- column-charset []
+ (first (jdbc/query db/*db-connection*
+ (str "SELECT collation_name AS `collation`, character_set_name AS `character-set` "
+ "FROM information_schema.`COLUMNS` "
+ "WHERE table_schema = 'utf8_test'"
+ " AND table_name = 'metabase_database'"
+ "AND column_name = 'name';"))))
+
+(def ^:private test-unicode-str "Cam ðŒ† Saul 💩")
+
+(defn- insert-row! []
+ (jdbc/execute! db/*db-connection* [(str "INSERT INTO metabase_database (engine, name, created_at, updated_at) "
+ "VALUES ('mysql', ?, current_timestamp, current_timestamp)")
+ test-unicode-str]))
+
+;; The basic idea behind this test is:
+;;
+;; 1. Create a new application DB; convert the DB to `latin1` or `utf8` (effectively rolling back migrations 107-160),
+;; then verify that utf-8 is now broken. (This simulates the state app DBs are in before this fix)
+;;
+;; 2. Now run the migrations again and verify that things are fixed
+(deftest utf8-test
+ (mt/test-driver :mysql
+ (testing "Migrations 107-160\n"
+ (doseq [{:keys [charset collation]} [{:charset "utf8", :collation "utf8_general_ci"}
+ {:charset "latin1", :collation "latin1_swedish_ci"}]]
+ ;; create a new application DB and run migrations.
+ (create-test-db!)
+ (jdbc/with-db-connection [conn-spec (test-db-spec)]
+ (mdb/migrate! conn-spec :up)
+ (testing (format "Migrating %s charset -> utf8mb4\n" charset)
+ ;; Roll back the DB to act as if migrations 107-160 had never been ran
+ (convert-to-charset! conn-spec charset collation)
+ (remove-utf8mb4-migrations! conn-spec)
+ (binding [db/*db-connection* conn-spec]
+ (testing (format "DB without migrations 107-160: UTF-8 shouldn't work when using the '%s' character set" charset)
+ (is (= {:character-set charset, :collation collation}
+ (db-charset)
+ (table-charset)
+ (column-charset))
+ (format "Make sure we converted the DB to %s correctly" charset))
+ (is (thrown?
+ Exception
+ (insert-row!))
+ "Shouldn't be able to insert UTF-8 values"))
+
+ (testing "If we run the migrations 107-160 then the DB should get converted to utf8mb4"
+ (mdb/migrate! conn-spec :up)
+ (is (= {:character-set "utf8mb4", :collation "utf8mb4_unicode_ci"}
+ (db-charset)
+ (table-charset)
+ (column-charset))
+ "DB should be converted back to `utf8mb4` after running migrations")
+ (testing "We should be able to insert UTF-8 values"
+ (insert-row!)
+ (is (= test-unicode-str
+ (db/select-one-field :name Database :name test-unicode-str))))))))))))
diff --git a/test/metabase/query_processor_test/date_bucketing_test.clj b/test/metabase/query_processor_test/date_bucketing_test.clj
index c2a97e52493..e08a9ec18e6 100644
--- a/test/metabase/query_processor_test/date_bucketing_test.clj
+++ b/test/metabase/query_processor_test/date_bucketing_test.clj
@@ -862,10 +862,10 @@
;; generating Java classes here so they'll be in the DB's native timezone. Some DBs refuse to use
;; the same timezone we're running the tests from *cough* SQL Server *cough*
[(u/prog1 (if (isa? driver/hierarchy driver/*driver* :sql)
- (driver/date-add driver/*driver*
- (sql.qp/current-datetime-fn driver/*driver*)
- (* i interval-seconds)
- :second)
+ (sql.qp/add-interval-honeysql-form driver/*driver*
+ (sql.qp/current-datetime-honeysql-form driver/*driver*)
+ (* i interval-seconds)
+ :second)
(u.date/add :second (* i interval-seconds)))
(assert <>))]))])))
--
GitLab