Serdes v2: Correctly (de)serialize references to segments and native snippets (#26881)

enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj

@@ -6,8 +6,9 @@
             [metabase-enterprise.serialization.v2.ingest :as serdes.ingest]
             [metabase-enterprise.serialization.v2.load :as serdes.load]
             [metabase.models :refer [Card Collection Dashboard DashboardCard Database Field FieldValues Metric
-                                     Segment Table Timeline TimelineEvent User]]
+                                     NativeQuerySnippet Segment Table Timeline TimelineEvent User]]
             [metabase.models.serialization.base :as serdes.base]
+            [metabase.util :as u]
             [schema.core :as s]
             [toucan.db :as db])
   (:import java.time.OffsetDateTime))
@@ -884,3 +885,43 @@
             (is (thrown-with-msg? clojure.lang.ExceptionInfo
                                   #"Failed to read file"
                                   (serdes.load/load-metabase ingestion)))))))))
+
+(deftest card-with-snippet-test
+  (let [db1s       (atom nil)
+        table1s    (atom nil)
+        snippet1s  (atom nil)
+        card1s     (atom nil)
+        extracted  (atom nil)]
+    (testing "snippets referenced by native cards must be deserialized"
+      (ts/with-empty-h2-app-db
+        (reset! db1s      (ts/create! Database :name "my-db"))
+        (reset! table1s   (ts/create! Table :name "CUSTOMERS" :db_id (:id @db1s)))
+        (reset! snippet1s (ts/create! NativeQuerySnippet :name "some snippet"))
+        (reset! card1s    (ts/create! Card
+                                      :name "the query"
+                                      :dataset_query {:database (:id @db1s)
+                                                      :native {:template-tags {"snippet: things"
+                                                                               {:id "e2d15f07-37b3-01fc-3944-2ff860a5eb46",
+                                                                                :name "snippet: filtered data",
+                                                                                :display-name "Snippet: Filtered Data",
+                                                                                :type :snippet,
+                                                                                :snippet-name "filtered data",
+                                                                                :snippet-id (:id @snippet1s)}}}}))
+        (ts/create! User :first_name "Geddy" :last_name "Lee"     :email "glee@rush.yyz")
+
+        (testing "on extraction"
+          (reset! extracted (serdes.base/extract-one "Card" {} @card1s))
+          (is (= (:entity_id @snippet1s)
+                 (-> @extracted :dataset_query :native :template-tags (get "snippet: things") :snippet-id))))
+
+        (testing "when loading"
+          (let [new-eid   (u/generate-nano-id)
+                ingestion (ingestion-in-memory [(assoc @extracted :entity_id new-eid)])]
+            (is (some? (serdes.load/load-metabase ingestion)))
+            (is (= (:id @snippet1s)
+                   (-> (db/select-one Card :entity_id new-eid)
+                       :dataset_query
+                       :native
+                       :template-tags
+                       (get "snippet: things")
+                       :snippet-id)))))))))

src/metabase/models/card.clj

@@ -407,13 +407,17 @@
 (defmethod serdes.base/serdes-descendants "Card" [_model-name id]
   (let [card          (db/select-one Card :id id)
         source-table  (some->  card :dataset_query :query :source-table)
-        template-tags (some->> card :dataset_query :native :template-tags vals (filter :card-id))]
+        template-tags (some->> card :dataset_query :native :template-tags vals (keep :card-id))
+        snippets      (some->> card :dataset_query :native :template-tags vals (keep :snippet-id))]
     (set/union
       (when (and (string? source-table)
                  (.startsWith ^String source-table "card__"))
         #{["Card" (Integer/parseInt (.substring ^String source-table 6))]})
       (when (seq template-tags)
-        (set (for [{:keys [card-id]} template-tags]
-               ["Card" card-id]))))))
+        (set (for [card-id template-tags]
+               ["Card" card-id])))
+      (when (seq snippets)
+        (set (for [snippet-id snippets]
+               ["NativeQuerySnippet" snippet-id]))))))
 
 (serdes.base/register-ingestion-path! "Card" (serdes.base/ingestion-matcher-collected "collections" "Card"))

src/metabase/models/serialization/util.clj

@@ -226,6 +226,9 @@
     mbql-entity-reference?
     (mbql-id->fully-qualified-name &match)
 
+    sequential?
+    (mapv ids->fully-qualified-names &match)
+
     map?
     (as-> &match entity
       (m/update-existing entity :database (fn [db-id]
@@ -240,14 +243,14 @@
                                                (mapv mbql-id->fully-qualified-name breakout)))
       (m/update-existing entity :aggregation (fn [aggregation]
                                                (mapv mbql-id->fully-qualified-name aggregation)))
-      (m/update-existing entity :filter      (fn [filter]
-                                               (m/map-vals mbql-id->fully-qualified-name filter)))
+      (m/update-existing entity :filter      ids->fully-qualified-names)
       (m/update-existing entity ::mb.viz/param-mapping-source export-field-fk)
+      (m/update-existing entity :segment    export-fk 'Segment)
       (m/update-existing entity :snippet-id export-fk 'NativeQuerySnippet)
       (merge entity
              (m/map-vals ids->fully-qualified-names
                          (dissoc entity
-                                 :database :card_id :card-id :source-table :breakout :aggregation :filter
+                                 :database :card_id :card-id :source-table :breakout :aggregation :filter :segment
                                  ::mb.viz/param-mapping-source :snippet-id))))))
 
 ;(ids->fully-qualified-names {:aggregation [[:sum [:field 277405 nil]]]})
@@ -321,7 +324,12 @@
     (_ :guard (every-pred map? (comp portable-id? :source_table)))
     (-> &match
         (assoc :source_table (str "card__" (import-fk (:source_table &match) 'Card)))
-        mbql-fully-qualified-names->ids*))) ;; process other keys
+        mbql-fully-qualified-names->ids*) ;; process other keys
+
+    (_ :guard (every-pred map? (comp portable-id? :snippet-id)))
+    (-> &match
+        (assoc :snippet-id (import-fk (:snippet-id &match) 'NativeQuerySnippet))
+        mbql-fully-qualified-names->ids*)))
 
 (defn- mbql-fully-qualified-names->ids
   [entity]
@@ -365,6 +373,7 @@
            (and (= k :source-table) (vector? v))      #{(table->path v)}
            (and (= k :source-table) (portable-id? v)) #{[{:model "Card" :id v}]}
            (and (= k :source-field) (vector? v))      #{(field->path v)}
+           (and (= k :snippet-id)   (portable-id? v)) #{[{:model "NativeQuerySnippet" :id v}]}
            (and (= k :card_id)      (string? v))      #{[{:model "Card" :id v}]}
            (and (= k :card-id)      (string? v))      #{[{:model "Card" :id v}]}
            (map? v)                                   (mbql-deps-map v)