Adds error reporting and --abort-on-error flag to import command (#28819)

Adds error reporting and --abort-on-error flag to v2 import command

enterprise/backend/src/metabase_enterprise/serialization/cmd.clj

@@ -71,15 +71,17 @@
         (throw e)))))
 
 (defn v2-load
-  "SerDes v2 load entry point"
-  [path]
+  "SerDes v2 load entry point.
+
+   opts are passed to load-metabase"
+  [path opts]
   (plugins/load-plugins!)
   (mdb/setup-db!)
   ; TODO This should be restored, but there's no manifest or other meta file written by v2 dumps.
   ;(when-not (load/compatible? path)
   ;  (log/warn (trs "Dump was produced using a different version of Metabase. Things may break!")))
   (log/info (trs "Loading serialized Metabase files from {0}" path))
-  (v2.load/load-metabase (v2.ingest/ingest-yaml path)))
+  (v2.load/load-metabase (v2.ingest/ingest-yaml path) opts))
 
 (defn- select-entities-in-collections
   ([model collections]

enterprise/backend/src/metabase_enterprise/serialization/v2/ingest/yaml.clj

@@ -7,8 +7,6 @@
    [metabase-enterprise.serialization.v2.ingest :as ingest]
    [metabase-enterprise.serialization.v2.utils.yaml :as u.yaml]
    [metabase.util.date-2 :as u.date]
-   [metabase.util.i18n :refer [trs]]
-   [metabase.util.log :as log]
    [yaml.core :as yaml]
    [yaml.reader :as y.reader])
   (:import
@@ -97,7 +95,6 @@
     (when-not @cache
       (reset! cache (ingest-all root-dir)))
     (let [{:keys [model id]} (first abs-path)]
-      (log/info (trs "Loading {0}" (u.yaml/log-path-str abs-path)))
       (if (and (= (count abs-path) 1)
                (= model "Setting"))
         {:serdes/meta abs-path :key (keyword id) :value (get settings (keyword id))}

enterprise/backend/src/metabase_enterprise/serialization/v2/load.clj

@@ -5,7 +5,10 @@
    [medley.core :as m]
    [metabase-enterprise.serialization.v2.backfill-ids :as serdes.backfill]
    [metabase-enterprise.serialization.v2.ingest :as serdes.ingest]
-   [metabase.models.serialization.base :as serdes.base]))
+   [metabase-enterprise.serialization.v2.utils.yaml :as u.yaml]
+   [metabase.models.serialization.base :as serdes.base]
+   [metabase.util.i18n :refer [trs]]
+   [metabase.util.log :as log]))
 
 (declare load-one)
 
@@ -40,6 +43,7 @@
 
   Circular dependencies are not allowed, and are detected and thrown as an error."
   [{:keys [expanding ingestion seen] :as ctx} path]
+  (log/info (trs "Loading {0}" (u.yaml/log-path-str path)))
   (cond
     (expanding path) (throw (ex-info (format "Circular dependency on %s" (pr-str path)) {:path path}))
     (seen path) ctx ; Already been done, just skip it.
@@ -69,15 +73,29 @@
                                  :deps-chain expanding}
                                 e)))))))
 
+(defn- try-load-one
+  [ctx path]
+  (try
+    (load-one ctx path)
+    (catch Exception e
+      (log/error (trs "Error importing {0}. Continuing..." (u.yaml/log-path-str path)))
+      (update ctx :errors conj e))))
+
 (defn load-metabase
   "Loads in a database export from an ingestion source, which is any Ingestable instance."
-  [ingestion]
+  [ingestion & {:keys [abort-on-error] :or {abort-on-error true}}]
   ;; We proceed in the arbitrary order of ingest-list, deserializing all the files. Their declared dependencies guide
   ;; the import, and make sure all containers are imported before contents, etc.
   (serdes.backfill/backfill-ids)
-  (let [contents (serdes.ingest/ingest-list ingestion)]
-    (reduce load-one {:expanding #{}
-                      :seen      #{}
-                      :ingestion ingestion
-                      :from-ids  (m/index-by :id contents)}
-            contents)))
+  (let [contents (serdes.ingest/ingest-list ingestion)
+        ctx      {:expanding #{}
+                  :seen      #{}
+                  :ingestion ingestion
+                  :from-ids  (m/index-by :id contents)
+                  :errors    []}
+        result   (reduce (if abort-on-error load-one try-load-one) ctx contents)]
+    (when-let [errors (seq (:errors result))]
+      (log/error (trs "Errors were encountered during import. Individual errors:"))
+      (doseq [e errors]
+        (log/error e)))
+    result))

src/metabase/cmd.clj

@@ -160,8 +160,9 @@
 
 (defn ^:command import
   "Load serialized Metabase instance as created by the [[export]] command from directory `path`."
-  [path]
-  (call-enterprise 'metabase-enterprise.serialization.cmd/v2-load path))
+  [path & options]
+  (let [opts {:abort-on-error (boolean (some #{"--abort-on-error"} options))}]
+    (call-enterprise 'metabase-enterprise.serialization.cmd/v2-load path opts)))
 
 (defn ^:command ^:deprecated dump
   "Deprecated: prefer [[export]] instead.

test/metabase/cmd_test.clj

@@ -23,7 +23,7 @@
                (cmd/load "/path/" "--num-cans" "2")))))
     (testing "import (v2)"
       (testing "with no options"
-        (is (= '(metabase-enterprise.serialization.cmd/v2-load "/path/")
+        (is (= '(metabase-enterprise.serialization.cmd/v2-load "/path/" {:abort-on-error false})
                (cmd/import "/path/")))))))
 
 (deftest export-test