serdes: do not die on unreadable yamls (#41887)

enterprise/backend/src/metabase_enterprise/serialization/v2/ingest.clj

@@ -8,6 +8,7 @@
    [clojure.string :as str]
    [metabase.models.serialization :as serdes]
    [metabase.util.date-2 :as u.date]
+   [metabase.util.log :as log]
    [metabase.util.yaml :as yaml]
    [potemkin.types :as p])
   (:import (java.io File)))
@@ -61,12 +62,17 @@
 (defn- ingest-all [^File root-dir]
   ;; This returns a map {unlabeled-hierarchy [original-hierarchy File]}.
   (into {} (for [^File file (file-seq root-dir)
-                 :when      (and (.isFile file)
-                                 (str/ends-with? (.getName file) ".yaml")
-                                 (let [rel (.relativize (.toPath root-dir) (.toPath file))]
-                                   (-> rel (.subpath 0 1) (.toString) legal-top-level-paths)))
+                 :when (and (.isFile file)
+                            (not (str/starts-with? (.getName file) "."))
+                            (str/ends-with? (.getName file) ".yaml")
+                            (let [rel (.relativize (.toPath root-dir) (.toPath file))]
+                              (-> rel (.subpath 0 1) (.toString) legal-top-level-paths)))
                  ;; TODO: only load YAML once.
-                 :let [hierarchy (serdes/path (ingest-file file))]]
+                 :let  [hierarchy (try
+                                    (serdes/path (ingest-file file))
+                                    (catch Exception e
+                                      (log/error e "Error reading file" (.getName file))))]
+                 :when hierarchy]
              [(strip-labels hierarchy) [hierarchy file]])))
 
 (deftype YamlIngestion [^File root-dir settings cache]

enterprise/backend/test/metabase_enterprise/serialization/v2/e2e_test.clj

@@ -856,3 +856,27 @@
                            (get-in viz [:column_settings
                                         (format "[\"ref\",[\"field\",%s,null]]" %people.name)
                                         :pivot_table.column_sort_order])))))))))))))
+
+(deftest extra-files-test
+  (testing "Adding some extra files does not break deserialization"
+    (ts/with-random-dump-dir [dump-dir "serdesv2-"]
+      (mt/with-empty-h2-app-db
+        (let [coll (ts/create! Collection :name "coll")
+              _    (ts/create! Card :name "card" :collection_id (:id coll))]
+          (storage/store! (extract/extract {:no-settings   true
+                                            :no-data-model true}) dump-dir)
+
+          (spit (io/file dump-dir "collections" ".hidden.yaml") "serdes/meta: [{do-not: read}]")
+          (spit (io/file dump-dir "collections" "unreadable.yaml") "\0")
+
+          (testing "No exceptions when loading despite unreadable files"
+            (let [logs (mt/with-log-messages-for-level ['metabase-enterprise :error]
+                         (let [files (->> (#'ingest/ingest-all (io/file dump-dir))
+                                          (map (comp second second))
+                                          (map #(.getName %))
+                                          set)]
+                           (testing "Hidden YAML wasn't read even though it's not throwing errors"
+                             (is (not (contains? files ".hidden.yaml"))))))]
+              (testing ".yaml files not containing valid yaml are just logged and do not break ingestion process"
+                (is (=? [[:error Throwable "Error reading file unreadable.yaml"]]
+                        logs))))))))))