diff --git a/resources/migrations/018_add_data_migrations_table.yaml b/resources/migrations/018_add_data_migrations_table.yaml new file mode 100644 index 0000000000000000000000000000000000000000..641e213296dee29202ff5f1e3831136b19589293 --- /dev/null +++ b/resources/migrations/018_add_data_migrations_table.yaml @@ -0,0 +1,25 @@ +databaseChangeLog: + - changeSet: + id: 18 + author: camsaul + changes: + - createTable: + tableName: data_migrations + columns: + - column: + name: id + type: varchar + constraints: + primaryKey: true + nullable: false + - column: + name: timestamp + type: DATETIME + constraints: + nullable: false + - createIndex: + tableName: data_migrations + indexName: idx_data_migrations_id + columns: + column: + name: id diff --git a/resources/migrations/liquibase.json b/resources/migrations/liquibase.json index 815d1513dc9eed314ee400bf161496b6d2afd20f..0baf1311681e58de0b48d399d286c766879c86a0 100644 --- a/resources/migrations/liquibase.json +++ b/resources/migrations/liquibase.json @@ -15,6 +15,7 @@ {"include": {"file": "migrations/014_add_view_log_table.yaml"}}, {"include": {"file": "migrations/015_add_revision_is_creation_field.yaml"}}, {"include": {"file": "migrations/016_user_last_login_allow_null.yaml"}}, - {"include": {"file": "migrations/017_add_database_is_sample_field.yaml"}} + {"include": {"file": "migrations/017_add_database_is_sample_field.yaml"}}, + {"include": {"file": "migrations/018_add_data_migrations_table.yaml"}} ] } diff --git a/src/metabase/db/migrations.clj b/src/metabase/db/migrations.clj index 792098c84c88fbdc2f8e8b5ac1cae9e50b997bc7..bc333ee5c0e10b62f2b55f0f334d82f2d916c794 100644 --- a/src/metabase/db/migrations.clj +++ b/src/metabase/db/migrations.clj @@ -1,28 +1,66 @@ (ns metabase.db.migrations + "Clojure-land data migration definitions and fns for running them." (:require [clojure.tools.logging :as log] [korma.core :as k] [metabase.db :as db] (metabase.models [card :refer [Card]] [database :refer [Database]] [setting :as setting]) - [metabase.sample-data :as sample-data])) + [metabase.sample-data :as sample-data] + [metabase.util :as u])) -(defn- set-card-database-and-table-ids - "Upgrade for the `Card` model when `:database_id`, `:table_id`, and `:query_type` were added and needed populating. +;;; # Migration Helpers - This reads through all saved cards, extracts the JSON from the `:dataset_query`, and tries to populate - the values for `:database_id`, `:table_id`, and `:query_type` if possible." +(defn- migration-ran? [migration] + (boolean (seq (k/select "data_migrations" + (k/where {:id (name migration)}) + (k/limit 1))))) + +(defn- run-migration-if-needed + "Run MIGRATION if needed. MIGRATION should be a symbol naming a fn that takes no arguments. + + (run-migration-if-needed 'set-card-database-and-table-ids)" + [migration] + (when-not (migration-ran? migration) + (log/info (format "Running data migration '%s'..." (name migration))) + (@(resolve migration)) + (k/insert "data_migrations" + (k/values {:id (name migration) + :timestamp (u/new-sql-timestamp)})) + (log/info "[ok]"))) + +(def ^:private data-migrations (atom [])) + +(defmacro ^:private defmigration + "Define a new data migration. This is just a simple wrapper around `defn-` that adds the function" + [migration-name & body] + `(do (defn- ~migration-name [] ~@body) + (swap! data-migrations conj '~migration-name))) + +(defn run-all + "Run all data migrations defined by `defmigration`." [] + (dorun (map run-migration-if-needed @data-migrations))) + + +;;; # Migration Definitions + +;; Upgrade for the `Card` model when `:database_id`, `:table_id`, and `:query_type` were added and needed populating. +;; +;; This reads through all saved cards, extracts the JSON from the `:dataset_query`, and tries to populate +;; the values for `:database_id`, `:table_id`, and `:query_type` if possible. +(defmigration set-card-database-and-table-ids ;; only execute when `:database_id` column on all cards is `nil` (when (= 0 (:cnt (first (k/select Card (k/aggregate (count :*) :cnt) (k/where (not= :database_id nil)))))) - (log/info "Data migration: Setting database/table/type fields on all Cards.") (doseq [{id :id {:keys [type] :as dataset-query} :dataset_query} (db/sel :many [Card :id :dataset_query])] (when type ;; simply resave the card with the dataset query which will automatically set the database, table, and type (db/upd Card id :dataset_query dataset-query))))) -(defn run-all - "Run all coded data migrations." - [] - ;; Append to the bottom of this list so that these run in chronological order - (set-card-database-and-table-ids)) + +;; Set the `:ssl` key in `details` to `false` for all existing MongoDB `Databases`. +;; UI was automatically setting `:ssl` to `true` for every database added as part of the auto-SSL detection. +;; Since Mongo did *not* support SSL, all existing Mongo DBs should actually have this key set to `false`. +(defmigration set-mongodb-databases-ssl-false + (doseq [{:keys [id details]} (db/sel :many :fields [Database :id :details] :engine "mongo")] + (db/upd Database id, :details (assoc details :ssl false))))