Skip to content
Snippets Groups Projects
Unverified Commit acdbf14d authored by john-metabase's avatar john-metabase Committed by GitHub
Browse files

Adds error reporting and --abort-on-error flag to import command (#28819)

Adds error reporting and --abort-on-error flag to v2 import command
parent eddc84b9
No related branches found
No related tags found
No related merge requests found
......@@ -71,15 +71,17 @@
(throw e)))))
(defn v2-load
"SerDes v2 load entry point"
[path]
"SerDes v2 load entry point.
opts are passed to load-metabase"
[path opts]
(plugins/load-plugins!)
(mdb/setup-db!)
; TODO This should be restored, but there's no manifest or other meta file written by v2 dumps.
;(when-not (load/compatible? path)
; (log/warn (trs "Dump was produced using a different version of Metabase. Things may break!")))
(log/info (trs "Loading serialized Metabase files from {0}" path))
(v2.load/load-metabase (v2.ingest/ingest-yaml path)))
(v2.load/load-metabase (v2.ingest/ingest-yaml path) opts))
(defn- select-entities-in-collections
([model collections]
......
......@@ -7,8 +7,6 @@
[metabase-enterprise.serialization.v2.ingest :as ingest]
[metabase-enterprise.serialization.v2.utils.yaml :as u.yaml]
[metabase.util.date-2 :as u.date]
[metabase.util.i18n :refer [trs]]
[metabase.util.log :as log]
[yaml.core :as yaml]
[yaml.reader :as y.reader])
(:import
......@@ -97,7 +95,6 @@
(when-not @cache
(reset! cache (ingest-all root-dir)))
(let [{:keys [model id]} (first abs-path)]
(log/info (trs "Loading {0}" (u.yaml/log-path-str abs-path)))
(if (and (= (count abs-path) 1)
(= model "Setting"))
{:serdes/meta abs-path :key (keyword id) :value (get settings (keyword id))}
......
......@@ -5,7 +5,10 @@
[medley.core :as m]
[metabase-enterprise.serialization.v2.backfill-ids :as serdes.backfill]
[metabase-enterprise.serialization.v2.ingest :as serdes.ingest]
[metabase.models.serialization.base :as serdes.base]))
[metabase-enterprise.serialization.v2.utils.yaml :as u.yaml]
[metabase.models.serialization.base :as serdes.base]
[metabase.util.i18n :refer [trs]]
[metabase.util.log :as log]))
(declare load-one)
......@@ -40,6 +43,7 @@
Circular dependencies are not allowed, and are detected and thrown as an error."
[{:keys [expanding ingestion seen] :as ctx} path]
(log/info (trs "Loading {0}" (u.yaml/log-path-str path)))
(cond
(expanding path) (throw (ex-info (format "Circular dependency on %s" (pr-str path)) {:path path}))
(seen path) ctx ; Already been done, just skip it.
......@@ -69,15 +73,29 @@
:deps-chain expanding}
e)))))))
(defn- try-load-one
[ctx path]
(try
(load-one ctx path)
(catch Exception e
(log/error (trs "Error importing {0}. Continuing..." (u.yaml/log-path-str path)))
(update ctx :errors conj e))))
(defn load-metabase
"Loads in a database export from an ingestion source, which is any Ingestable instance."
[ingestion]
[ingestion & {:keys [abort-on-error] :or {abort-on-error true}}]
;; We proceed in the arbitrary order of ingest-list, deserializing all the files. Their declared dependencies guide
;; the import, and make sure all containers are imported before contents, etc.
(serdes.backfill/backfill-ids)
(let [contents (serdes.ingest/ingest-list ingestion)]
(reduce load-one {:expanding #{}
:seen #{}
:ingestion ingestion
:from-ids (m/index-by :id contents)}
contents)))
(let [contents (serdes.ingest/ingest-list ingestion)
ctx {:expanding #{}
:seen #{}
:ingestion ingestion
:from-ids (m/index-by :id contents)
:errors []}
result (reduce (if abort-on-error load-one try-load-one) ctx contents)]
(when-let [errors (seq (:errors result))]
(log/error (trs "Errors were encountered during import. Individual errors:"))
(doseq [e errors]
(log/error e)))
result))
......@@ -160,8 +160,9 @@
(defn ^:command import
"Load serialized Metabase instance as created by the [[export]] command from directory `path`."
[path]
(call-enterprise 'metabase-enterprise.serialization.cmd/v2-load path))
[path & options]
(let [opts {:abort-on-error (boolean (some #{"--abort-on-error"} options))}]
(call-enterprise 'metabase-enterprise.serialization.cmd/v2-load path opts)))
(defn ^:command ^:deprecated dump
"Deprecated: prefer [[export]] instead.
......
......@@ -23,7 +23,7 @@
(cmd/load "/path/" "--num-cans" "2")))))
(testing "import (v2)"
(testing "with no options"
(is (= '(metabase-enterprise.serialization.cmd/v2-load "/path/")
(is (= '(metabase-enterprise.serialization.cmd/v2-load "/path/" {:abort-on-error false})
(cmd/import "/path/")))))))
(deftest export-test
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment