Port down migration and H2 dump loading test from 47 to 48 (#35805)

src/metabase/config.clj

@@ -4,8 +4,7 @@
    [clojure.java.io :as io]
    [clojure.string :as str]
    [environ.core :as env]
-   [metabase.plugins.classloader :as classloader]
-   [metabase.util.log :as log])
+   [metabase.plugins.classloader :as classloader])
   (:import
    (clojure.lang Keyword)))
 
@@ -121,6 +120,13 @@
    Looks something like `Metabase v0.25.0.RC1`."
   (str "Metabase " (mb-version-info :tag)))
 
+(defn current-major-version
+  "Returns the major version of the running Metabase JAR.
+  When the version.properties file is missing (e.g., running in local dev), returns nil."
+  []
+  (some-> (second (re-find #"\d+\.(\d+)" (:tag mb-version-info)))
+          parse-long))
+
 (defonce ^{:doc "This UUID is randomly-generated upon launch and used to identify this specific Metabase instance during
                 this specifc run. Restarting the server will change this UUID, and each server in a horizontal cluster
                 will have its own ID, making this different from the `site-uuid` Setting."}
@@ -132,21 +138,6 @@
   mb-version-and-process-identifier
   (format "%s [%s]" mb-app-id-string local-process-uuid))
 
-;; In 0.41.0 we switched from Leiningen to deps.edn. This warning here to keep people from being bitten in the ass by
-;; the little gotcha described below.
-;;
-;; TODO -- after we've shipped 0.43.0, remove this warning. At that point, the last three shipped major releases will
-;; all be deps.edn based.
-(when (and (not is-prod?)
-           (.exists (io/file ".lein-env")))
-  ;; don't need to i18n since this is a dev-only warning.
-  (log/warn
-   (str "Found .lein-env in the project root directory.\n"
-        "This file was previously created automatically by the Leiningen lein-env plugin.\n"
-        "Environ will use values from it in preference to env var or Java system properties you've specified.\n"
-        "You should delete it; it will be recreated as needed when switching to a branch still using Leiningen.\n"
-        "See https://github.com/metabase/metabase/wiki/Migrating-from-Leiningen-to-tools.deps#custom-env-var-values for more details.")))
-
 (defn mb-user-defaults
   "Default user details provided as a JSON string at launch time for first-user setup flow."
   []

src/metabase/db/liquibase.clj

@@ -87,17 +87,22 @@
                                first
                                :id)]
      (cond
-      (nil? latest-migration)
-      changelog-file
-      ;; pre 42
-      (not (str/starts-with? latest-migration "v"))
-      changelog-legacy-file
+       (nil? latest-migration)
+       changelog-file
 
-      (< (->> latest-migration (re-find #"v(\d+)\..*") second parse-long) 45)
-      changelog-legacy-file
+       ;; post-44 installation downgraded to 45
+       (= latest-migration "v00.00-000")
+       changelog-file
 
-      :else
-      changelog-file))))
+       ;; pre 42
+       (not (str/starts-with? latest-migration "v"))
+       changelog-legacy-file
+
+       (< (->> latest-migration (re-find #"v(\d+)\..*") second parse-long) 45)
+       changelog-legacy-file
+
+       :else
+       changelog-file))))
 
 (defn- liquibase-connection ^JdbcConnection [^java.sql.Connection jdbc-connection]
   (JdbcConnection. jdbc-connection))
@@ -324,24 +329,19 @@
   [s]
   (map #(Integer/parseInt %) (re-seq #"\d+" s)))
 
-(defn- current-major-version
-  "Returns the major version of the running Metabase JAR"
-  []
-  (second (extract-numbers (:tag config/mb-version-info))))
-
 (defn rollback-major-version
   "Roll back migrations later than given Metabase major version"
   ;; default rollback to previous version
   ([db-type conn liquibase]
    ;; get current major version of Metabase we are running
-   (rollback-major-version db-type conn liquibase (dec (current-major-version))))
+   (rollback-major-version db-type conn liquibase (dec (config/current-major-version))))
 
   ;; with explicit target version
   ([_db-type conn ^Liquibase liquibase target-version]
    (when (or (not (integer? target-version)) (< target-version 44))
      (throw (IllegalArgumentException.
              (format "target version must be a number between 44 and the previous major version (%d), inclusive"
-                     (current-major-version)))))
+                     (config/current-major-version)))))
    ;; count and rollback only the applied change set ids which come after the target version (only the "v..." IDs need to be considered)
    (let [changeset-query (format "SELECT id FROM %s WHERE id LIKE 'v%%' ORDER BY ORDEREXECUTED ASC" (changelog-table-name conn))
          changeset-ids   (map :id (jdbc/query {:connection conn} [changeset-query]))

test/metabase/cmd/load_from_h2_test.clj

@@ -2,16 +2,25 @@
   (:require
    [clojure.test :refer :all]
    [metabase.cmd.copy :as copy]
+   [metabase.cmd.dump-to-h2 :as dump-to-h2]
    [metabase.cmd.load-from-h2 :as load-from-h2]
    [metabase.cmd.test-util :as cmd.test-util]
    [metabase.db.connection :as mdb.connection]
+   [metabase.db.liquibase :as liquibase]
+   [metabase.db.setup :as mdb.setup]
    [metabase.db.test-util :as mdb.test-util]
    [metabase.driver :as driver]
    [metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]
    [metabase.models :refer [Table]]
    [metabase.test :as mt]
    [metabase.test.data.interface :as tx]
-   [toucan2.core :as t2]))
+   [metabase.util.log :as log]
+   [toucan2.connection :as t2.conn]
+   [toucan2.core :as t2])
+  (:import
+   (liquibase.changelog ChangeSet)))
+
+(set! *warn-on-reflection* true)
 
 (defn- load-from-h2-test* [db-name thunk]
   ;; enable this test in the REPL with something like (mt/set-test-drivers! #{:postgres})
@@ -48,3 +57,98 @@
        (testing "H2 connection details SHOULD have been copied"
          (is (=? {:db string?}
                 (t2/select-one-fn :details :model/Database :engine :h2))))))))
+
+(defn- get-data-source [db-type db-def]
+  (let [connection-details (tx/dbdef->connection-details db-type :db db-def)
+        db-spec (sql-jdbc.conn/connection-details->spec db-type connection-details)]
+    (mdb.test-util/->ClojureJDBCSpecDataSource db-spec)))
+
+(defn- create-current-database
+  [db-type db-def data-source]
+  (tx/create-db! db-type db-def)
+  (mdb.setup/setup-db! db-type data-source true))
+
+(defn- dump-filename
+  [h2-filename version]
+  (str h2-filename "-" version))
+
+(defn- liquibase-latest-major-version
+  []
+  (t2.conn/with-connection [conn]
+    (liquibase/with-liquibase [liquibase conn]
+      (let [change-sets (.. liquibase getDatabaseChangeLog getChangeSets)
+            size (count change-sets)]
+        (if (pos? size)
+          (let [change-set-id (.getId ^ChangeSet (.get change-sets (dec size)))
+                [_ major] (re-find #"v(\d+).*" change-set-id)]
+            (if major
+              (parse-long major)
+              (throw (ex-info "couldn't parse major version from change-set-id " change-set-id
+                              {:change-set-id change-set-id}))))
+          (throw (ex-info "no changesets found" {})))))))
+
+(def ^:private current-major-version
+  ;; We are interested in the latest version we started preparing
+  ;; and we assume that every version has database migrations.
+  ;; (Downgrading and upgrading between versions with identical
+  ;; databases is trivial, so the difference is probably not really
+  ;; interesting.)
+  (delay (liquibase-latest-major-version)))
+
+(defn- migrate-down-then-up-and-create-dump
+  [db-name h2-filename version]
+  (let [db-type driver/*driver*
+        db-def {:database-name db-name}
+        current-version @current-major-version
+        data-source (get-data-source db-type db-def)]
+    (log/info "creating database")
+    (create-current-database db-type db-def data-source)
+    (binding [mdb.connection/*application-db* (mdb.connection/application-db db-type data-source)]
+      (mt/dataset bird-flocks
+        ;; make sure the data is there
+        (is (= 18 (ffirst (mt/formatted-rows [int]
+                                             (mt/run-mbql-query bird
+                                               {:aggregation [[:count]]})))))
+        (let [filename (dump-filename h2-filename version)]
+          (when (< version current-version)
+            (log/info "rolling back to version" version)
+            (t2.conn/with-connection [conn]
+              (liquibase/with-liquibase [liquibase conn]
+                (liquibase/rollback-major-version db-type conn liquibase version))))
+          (log/info "creating dump" filename)
+          ;; this migrates the DB back to the newest and creates a dump
+          (dump-to-h2/dump-to-h2! filename)
+          ;; check if after a down and up migration we can still run a query
+          (is (= 18 (ffirst (mt/formatted-rows [int]
+                                               (mt/run-mbql-query bird
+                                                 {:aggregation [[:count]]}))))))))))
+
+(defn- load-dump
+  [db-name h2-filename version]
+  (let [db-type driver/*driver*
+        db-def {:database-name db-name}
+        data-source (get-data-source db-type db-def)]
+    (create-current-database db-type db-def data-source)
+    (binding [mdb.connection/*application-db* (mdb.connection/application-db db-type data-source)]
+      (mt/dataset sad-toucan-incidents
+        (is (= 200 (ffirst (mt/formatted-rows [int]
+                             (mt/run-mbql-query incidents
+                               {:aggregation [[:count]]})))))
+        (log/info "loading dump" h2-filename "version" version)
+        (load-from-h2/load-from-h2! (dump-filename h2-filename version))
+        ;; check that we can run the query using data from the dump
+        (is (= 18 (ffirst (mt/formatted-rows [int]
+                            (mt/run-mbql-query bird
+                              {:aggregation [[:count]]})))))))))
+
+(deftest down-migrate-and-load-dump-test
+  (mt/test-drivers #{:mysql :postgres}
+    (mt/with-temp-dir [dir nil]
+      (let [h2-filename (str dir "/dump")
+            current-version (or @current-major-version
+                                (throw (ex-info "Couldn't determine current major version" {})))
+            supported-downgrades 4
+            versions (range current-version (- current-version supported-downgrades) -1)]
+        (doseq [version versions]
+          (migrate-down-then-up-and-create-dump "load-test-source" h2-filename version)
+          (load-dump "load-test-target" h2-filename version))))))