backported "[serdes-v1] can now dump/load entity_id" (#39524)

Co-authored-by: Alexander Solovyov <alexander@solovyov.net>

enterprise/backend/src/metabase_enterprise/serialization/cmd.clj

@@ -5,6 +5,7 @@
    [clojure.string :as str]
    [metabase-enterprise.serialization.dump :as dump]
    [metabase-enterprise.serialization.load :as load]
+   [metabase-enterprise.serialization.serialize :as serialize]
    [metabase-enterprise.serialization.v2.entity-ids :as v2.entity-ids]
    [metabase-enterprise.serialization.v2.extract :as v2.extract]
    [metabase-enterprise.serialization.v2.ingest :as v2.ingest]
@@ -181,7 +182,7 @@
 
 (defn v1-dump!
   "Legacy Metabase app data dump"
-  [path {:keys [state user] :or {state :active} :as opts}]
+  [path {:keys [state user include-entity-id] :or {state :active} :as opts}]
   (log/info (trs "BEGIN DUMP to {0} via user {1}" path user))
   (mdb/setup-db!)
   (check-premium-token!)
@@ -206,18 +207,19 @@
                       (t2/select Metric :table_id [:in (map :id tables)] {:order-by [[:id :asc]]})
                       (t2/select Metric))
         collections (select-collections users state)]
-    (dump/dump! path
-               databases
-               tables
-               (mapcat field/with-values (u/batches-of 32000 fields))
-               metrics
-               (select-segments-in-tables tables state)
-               collections
-               (select-entities-in-collections NativeQuerySnippet collections state)
-               (select-entities-in-collections Card collections state)
-               (select-entities-in-collections Dashboard collections state)
-               (select-entities-in-collections Pulse collections state)
-               users))
+    (binding [serialize/*include-entity-id* (boolean include-entity-id)]
+      (dump/dump! path
+                  databases
+                  tables
+                  (mapcat field/with-values (u/batches-of 32000 fields))
+                  metrics
+                  (select-segments-in-tables tables state)
+                  collections
+                  (select-entities-in-collections NativeQuerySnippet collections state)
+                  (select-entities-in-collections Card collections state)
+                  (select-entities-in-collections Dashboard collections state)
+                  (select-entities-in-collections Pulse collections state)
+                  users)))
   (dump/dump-settings! path)
   (dump/dump-dimensions! path)
   (log/info (trs "END DUMP to {0} via user {1}" path user)))

enterprise/backend/src/metabase_enterprise/serialization/serialize.clj

@@ -36,6 +36,10 @@
   ;; version 2 - start adding namespace portion to /collections/ paths
   2)
 
+(def ^:dynamic *include-entity-id*
+  "If entity_id should be included in v1 serialization dump"
+  false)
+
 (def ^:private ^{:arglists '([form])} mbql-entity-reference?
   "Is given form an MBQL entity reference?"
   (partial mbql.normalize/is-clause? #{:field :field-id :fk-> :metric :segment}))
@@ -103,7 +107,8 @@
   [entity]
   (cond-> (dissoc entity :id :creator_id :created_at :updated_at :db_id :location
                   :dashboard_id :fields_hash :personal_owner_id :made_public_by_id :collection_id
-                  :pulse_id :result_metadata :entity_id :action_id)
+                  :pulse_id :result_metadata :action_id)
+    (not *include-entity-id*)   (dissoc :entity_id)
     (some #(instance? % entity) (map type [Metric Field Segment])) (dissoc :table_id)))
 
 (defmulti ^:private serialize-one

enterprise/backend/test/metabase_enterprise/serialization/cmd_test.clj

@@ -241,3 +241,42 @@
                              ;; t2/with-transactions re-wraps errors with data about toucan connections
                              "error_message" #".*Cannot import Collection.*"}
                             (-> (snowplow-test/pop-event-data-and-user-id!) first :data)))))))))))))
+
+(deftest entity-id-dump&load-test
+  (let [entity-ids* (atom {})
+        eid-map     #(into {} (map (juxt :name :entity_id) %))]
+    (testing "--include-entity-id should include entity ids in serialization"
+      (mt/with-premium-features #{:serialization}
+        (ts/with-random-dump-dir [dump-dir "serialization"]
+          (ts/with-dbs [source-db dest-db]
+             (testing "create 2 questions in a dashboard"
+               (ts/with-db source-db
+                 (let [db   (ts/create! Database)
+                       dash (ts/create! Dashboard)
+                       c1   (ts/create! Card {:name          "card1"
+                                              :database_id   (:id db)
+                                              :dataset_query {:database (:id db), :type :native, :native {:query "SELECT 1;"}}})
+                       c2   (ts/create! Card {:name          "card2"
+                                              :database_id   (:id db)
+                                              :dataset_query {:database (:id db), :type :native, :native {:query "SELECT 1;"}}})
+                       _    (ts/create! DashboardCard {:dashboard_id (:id dash) :card_id (:id c1)})
+                       _    (ts/create! DashboardCard {:dashboard_id (:id dash) :card_id (:id c2)})]
+                   (testing "initial dump"
+                     (is (nil? (cmd/dump dump-dir))))
+                   (testing "storing original entity ids"
+                     (is (reset! entity-ids* (eid-map [c1 c2])))))))
+             (testing "initial load"
+               (ts/with-db dest-db
+                 (is (some? (ts/create! User, :is_superuser true)))
+                 (is (nil? (cmd/load dump-dir "--on-error" "abort")))
+                 (testing "verify that entities got their own entity_id"
+                   (is (not= @entity-ids*
+                             (eid-map (t2/select Card)))))))
+             (testing "creating dump with entity ids included"
+               (ts/with-db source-db
+                 (is (nil? (cmd/dump dump-dir "--include-entity-id")))))
+             (testing "loading dump with entity ids will overwrite new entity ids with original ones"
+               (ts/with-db dest-db
+                 (is (nil? (cmd/load dump-dir "--on-error" "abort" "--mode" "update")))
+                 (is (= @entity-ids*
+                        (eid-map (t2/select Card))))))))))))

src/metabase/cmd.clj

@@ -198,7 +198,8 @@
                :default      :all
                :default-desc "all"
                :parse-fn     mbql.u/normalize-token
-               :validate     [#{:active :all} "Must be 'active' or 'all'"]]]}
+               :validate     [#{:active :all} "Must be 'active' or 'all'"]]
+              [nil "--include-entity-id"   "Include entity_id property in all dumped entities. Default: false."]]}
   [path & options]
   (log/warn (u/colorize :red (trs "''dump'' is deprecated and will be removed in a future release. Please migrate to ''export''.")))
   (call-enterprise 'metabase-enterprise.serialization.cmd/v1-dump! path (get-parsed-options #'dump options)))