Merge pull request #1393 from metabase/data-migrations

`defmigration` macro for defining data migrations

resources/migrations/018_add_data_migrations_table.yaml

+databaseChangeLog:
+  - changeSet:
+      id: 18
+      author: camsaul
+      changes:
+        - createTable:
+            tableName: data_migrations
+            columns:
+              - column:
+                  name: id
+                  type: VARCHAR
+                  constraints:
+                    primaryKey: true
+                    nullable: false
+              - column:
+                  name: timestamp
+                  type: DATETIME
+                  constraints:
+                    nullable: false
+        - createIndex:
+            tableName: data_migrations
+            indexName: idx_data_migrations_id
+            columns:
+              column:
+                name: id

resources/migrations/liquibase.json

@@ -15,6 +15,7 @@
       {"include": {"file": "migrations/014_add_view_log_table.yaml"}},
       {"include": {"file": "migrations/015_add_revision_is_creation_field.yaml"}},
       {"include": {"file": "migrations/016_user_last_login_allow_null.yaml"}},
-      {"include": {"file": "migrations/017_add_database_is_sample_field.yaml"}}
+      {"include": {"file": "migrations/017_add_database_is_sample_field.yaml"}},
+      {"include": {"file": "migrations/018_add_data_migrations_table.yaml"}}
   ]
 }

src/metabase/db.clj

@@ -95,6 +95,7 @@
 
    *  `:up`            - Migrate up
    *  `:down`          - Rollback *all* migrations
+   *  `:down-one`      - Rollback a single migration
    *  `:print`         - Just print the SQL for running the migrations, don't actually run them.
    *  `:release-locks` - Manually release migration locks left by an earlier failed migration.
                          (This shouldn't be necessary now that we run migrations inside a transaction,
@@ -110,6 +111,7 @@
          (case direction
            :up            (.update liquibase "")
            :down          (.rollback liquibase 10000 "")
+           :down-one      (.rollback liquibase 1 "")
            :print         (let [writer (StringWriter.)]
                             (.update liquibase "" writer)
                             (.toString writer))

src/metabase/db/migrations.clj

 (ns metabase.db.migrations
+  "Clojure-land data migration definitions and fns for running them."
   (:require [clojure.tools.logging :as log]
             [korma.core :as k]
             [metabase.db :as db]
             (metabase.models [card :refer [Card]]
                              [database :refer [Database]]
                              [setting :as setting])
-            [metabase.sample-data :as sample-data]))
+            [metabase.sample-data :as sample-data]
+            [metabase.util :as u]))
 
-(defn- set-card-database-and-table-ids
-  "Upgrade for the `Card` model when `:database_id`, `:table_id`, and `:query_type` were added and needed populating.
+;;; # Migration Helpers
 
-   This reads through all saved cards, extracts the JSON from the `:dataset_query`, and tries to populate
-   the values for `:database_id`, `:table_id`, and `:query_type` if possible."
+(defn- migration-ran? [migration-name]
+  (-> (k/select :data_migrations
+                (k/aggregate (count :*) :count)
+                (k/where {:id (name migration-name)}))
+      first :count (> 0)))
+
+(defn- run-migration-if-needed
+  "Run migration defined by MIGRATION-VAR if needed.
+
+     (run-migration-if-needed #'set-card-database-and-table-ids)"
+  [migration-var]
+  (let [migration-name (name (:name (meta migration-var)))]
+    (when-not (migration-ran? migration-name)
+      (log/info (format "Running data migration '%s'..." migration-name))
+      (@migration-var)
+      (k/insert "data_migrations"
+                (k/values {:id        migration-name
+                           :timestamp (u/new-sql-timestamp)}))
+      (log/info "[ok]"))))
+
+(def ^:private data-migrations (atom []))
+
+(defmacro ^:private defmigration
+  "Define a new data migration. This is just a simple wrapper around `defn-` that adds the resulting var to that `data-migrations` atom."
+  [migration-name & body]
+  `(do (defn- ~migration-name [] ~@body)
+       (swap! data-migrations conj #'~migration-name)))
+
+(defn run-all
+  "Run all data migrations defined by `defmigration`."
   []
+  (dorun (map run-migration-if-needed @data-migrations)))
+
+
+;;; # Migration Definitions
+
+;; Upgrade for the `Card` model when `:database_id`, `:table_id`, and `:query_type` were added and needed populating.
+;;
+;; This reads through all saved cards, extracts the JSON from the `:dataset_query`, and tries to populate
+;; the values for `:database_id`, `:table_id`, and `:query_type` if possible.
+(defmigration set-card-database-and-table-ids
   ;; only execute when `:database_id` column on all cards is `nil`
   (when (= 0 (:cnt (first (k/select Card (k/aggregate (count :*) :cnt) (k/where (not= :database_id nil))))))
-    (log/info "Data migration: Setting database/table/type fields on all Cards.")
     (doseq [{id :id {:keys [type] :as dataset-query} :dataset_query} (db/sel :many [Card :id :dataset_query])]
       (when type
         ;; simply resave the card with the dataset query which will automatically set the database, table, and type
         (db/upd Card id :dataset_query dataset-query)))))
 
-(defn run-all
-  "Run all coded data migrations."
-  []
-  ;; Append to the bottom of this list so that these run in chronological order
-  (set-card-database-and-table-ids))
+
+;; Set the `:ssl` key in `details` to `false` for all existing MongoDB `Databases`.
+;; UI was automatically setting `:ssl` to `true` for every database added as part of the auto-SSL detection.
+;; Since Mongo did *not* support SSL, all existing Mongo DBs should actually have this key set to `false`.
+(defmigration set-mongodb-databases-ssl-false
+  (doseq [{:keys [id details]} (db/sel :many :fields [Database :id :details] :engine "mongo")]
+    (db/upd Database id, :details (assoc details :ssl false))))