Skip to content
Snippets Groups Projects
Unverified Commit 55ed5e0f authored by Alexander Solovyov's avatar Alexander Solovyov Committed by GitHub
Browse files

Stable order of serialization keys (#35485) 

keys in serialization are be sorted by historical order first and alphabetically if they are not mentioned
parent 4d91483b
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
enterprise/backend/src/metabase_enterprise/serialization/dump.clj
+ 33
− 2
View file @ 55ed5e0f
(ns metabase-enterprise.serialization.dump
"Serialize entities into a directory structure of YAMLs."
(:require
[clojure.edn :as edn]
[clojure.java.io :as io]
[metabase-enterprise.serialization.names
:refer [fully-qualified-name name-for-logging safe-name]]
......@@ -17,7 +18,6 @@
[metabase.models.setting :as setting]
[metabase.models.table :refer [Table]]
[metabase.models.user :refer [User]]
[metabase.util :as u]
[metabase.util.i18n :as i18n :refer [trs]]
[metabase.util.log :as log]
[metabase.util.yaml :as yaml]
......@@ -25,13 +25,44 @@
(set! *warn-on-reflection* true)
(def ^:private serialization-order
(delay (-> (edn/read-string (slurp (io/resource "serialization-order.edn")))
(update-vals (fn [order]
(into {} (map vector order (range))))))))
(defn- serialization-sorted-map* [order-key]
(if-let [order (or (get @serialization-order order-key)
(get @serialization-order (last order-key)))]
;; known columns are sorted by their order, then unknown are sorted alphabetically
(let [getter #(if (contains? order %)
[0 (get order %)]
[1 %])]
(sorted-map-by (fn [k1 k2]
(compare (getter k1) (getter k2)))))
(sorted-map)))
(def ^:private serialization-sorted-map (memoize serialization-sorted-map*))
(defn- serialization-deep-sort
([m]
(let [model (-> (:serdes/meta m) last :model)]
(serialization-deep-sort m [(keyword model)])))
([m path]
(into (serialization-sorted-map path)
(for [[k v] m]
[k (cond
(map? v) (serialization-deep-sort v (conj path k))
(and (sequential? v)
(map? (first v))) (mapv #(serialization-deep-sort % (conj path k)) v)
:else v)]))))
(defn spit-yaml
"Writes obj to filename and creates parent directories if necessary.
Writes (even nested) yaml keys in a deterministic fashion."
[filename obj]
(io/make-parents filename)
(spit filename (yaml/generate-string (u/deep-sort-map obj)
(spit filename (yaml/generate-string (serialization-deep-sort obj)
{:dumper-options {:flow-style :block :split-lines false}})))
(defn- as-file?
......
enterprise/backend/test/metabase_enterprise/serialization/v2/storage_test.clj
+ 57
− 2
View file @ 55ed5e0f
(ns ^:mb/once metabase-enterprise.serialization.v2.storage-test
(:require
[clojure.java.io :as io]
[clojure.string :as str]
[clojure.test :refer :all]
[java-time.api :as t]
[metabase-enterprise.serialization.dump :as dump]
[metabase-enterprise.serialization.test-util :as ts]
[metabase-enterprise.serialization.v2.extract :as extract]
[metabase-enterprise.serialization.v2.storage :as storage]
[metabase.models :refer [Card Collection Dashboard Database Field FieldValues NativeQuerySnippet Table]]
[metabase.models :refer [Card Collection Dashboard DashboardCard Database Field FieldValues NativeQuerySnippet
Table]]
[metabase.models.serialization :as serdes]
[metabase.test :as mt]
[metabase.util.date-2 :as u.date]
......@@ -72,7 +75,7 @@
Card c3 {:name "parent card" :collection_id (:id parent)}
Card c4 {:name "child card" :collection_id (:id child)}
Dashboard d1 {:name "parent dash" :collection_id (:id parent)}]
(let [export (into [] (extract/extract nil))]
(let [export (into [] (extract/extract nil))]
(storage/store! export dump-dir)
(testing "the right files in the right places"
(let [gp-dir (str (:entity_id grandparent) "_grandparent_collection")
......@@ -88,6 +91,7 @@
[gp-dir p-dir "dashboards" (str (:entity_id d1) "_parent_dash.yaml")]} ; Parent dashboard
(file-set (io/file dump-dir "collections")))))))))))
(deftest snippets-collections-nesting-test
(ts/with-random-dump-dir [dump-dir "serdesv2-"]
(mt/with-empty-h2-app-db
......@@ -150,3 +154,54 @@
"fields" "Company__SLASH__organization website.yaml"))
(update :visibility_type keyword)
(update :base_type keyword))))))))))
(deftest yaml-sorted-test
(mt/with-empty-h2-app-db
(ts/with-temp-dpc [Database db {:name "My Company Data"}
Table t {:name "Customers" :db_id (:id db)}
Field w {:name "Company/organization website" :table_id (:id t)}
FieldValues _ {:field_id (:id w)}
Collection col {:name "Some Collection"}
Card c1 {:name "some card" :collection_id nil}
Card c2 {:name "other card" :collection_id (:id col)}
Dashboard d1 {:name "some dash" :collection_id (:id col)}
DashboardCard _ {:card_id (:id c1) :dashboard_id (:id d1)}
DashboardCard _ {:card_id (:id c2) :dashboard_id (:id d1)}
NativeQuerySnippet _ {:name "root snippet" :collection_id nil}]
(let [export (extract/extract nil)
check-sort (fn [coll order]
(loop [[k :as ks] (keys coll)
idx -1]
(let [new-idx (get order k)]
(if (nil? new-idx)
;; rest are sorted alphabetically
(is (= (not-empty (sort ks))
(not-empty ks)))
(do
;; check every present key is sorted in a monotone increasing order
(is (< idx (get order k)))
(recur (rest ks)
(long new-idx)))))))
descend (fn descend
([coll]
(let [model (-> (:serdes/meta coll) last :model)]
(is model)
(descend coll [(keyword model)])))
([coll path]
(let [order (or (get @@#'dump/serialization-order path)
(get @@#'dump/serialization-order (last path)))]
(testing (str "Path = " path)
(is order)
(check-sort coll order))
(doseq [[k v] coll]
(cond
(map? v) (descend v (conj path k))
(and (sequential? v)
(map? (first v))) (run! #(descend % (conj path k)) v))))))]
(with-redefs [spit (fn [fname yaml-data]
(testing (format "File %s\n" fname)
(let [coll (yaml/parse-string yaml-data)]
(if (str/ends-with? fname "settings.yaml")
(descend coll [:settings])
(descend coll)))))]
(storage/store! export "/non-existent"))))))
resources/serialization-order.edn 0 → 100644
+ 157
− 0
View file @ 55ed5e0f
{:settings []
:serdes/meta []
:Card [:name
:description
:entity_id
:created_at
:creator_id
:display
:archived
:collection_id
:collection_preview
:collection_position
:query_type
:dataset
:cache_ttl
:database_id
:table_id
:enable_embedding
:embedding_params
:made_public_by_id
:public_uuid
:parameters
:parameter_mappings
:dataset_query
:result_metadata
:visualization_settings
:serdes/meta]
[:Card :dataset_query] []
[:Card :visualization_settings] []
:Collection [:name
:description
:entity_id
:slug
:created_at
:archived
:type
:parent_id
:personal_owner_id
:namespace
:authority_level
:serdes/meta]
:Dashboard [:name
:description
:entity_id
:created_at
:creator_id
:archived
:collection_id
:auto_apply_filters
:cache_ttl
:collection_position
:position
:enable_embedding
:embedding_params
:made_public_by_id
:public_uuid
:show_in_getting_started
:caveats
:points_of_interest
:parameters
:ordered_cards
:ordered_tabs
:serdes/meta]
[:Dashboard :dashcards] [:entity_id
:card_id
:created_at
:row
:col
:size_x
:size_y]
[:Dashboard
:dashcards
:visualization_settings] []
:Database [:name
:description
:engine
:dbms_version
:created_at
:creator_id
:timezone
:auto_run_queries
:cache_ttl
:refingerprint
:is_full_sync
:is_on_demand
:is_sample
:is_audit
:metadata_sync_schedule
:cache_field_values_schedule
:settings
:options
:caveats
:points_of_interest
:initial_sync_status
:details
:serdes/meta]
[:Database :dbms_version] [:flavor
:version]
:FieldValue [:created_at
:hash_key
:has_more_values
:last_used_at
:type
:human_readable_values
:values
:serdes/meta]
:Field [:name
:display_name
:description
:created_at
:active
:visibility_type
:table_id
:database_type
:base_type
:effective_type
:semantic_type
:database_is_auto_increment
:database_required
:fk_target_field_id
:dimensions
:json_unfolding
:parent_id
:coercion_strategy
:preview_display
:position
:custom_position
:database_position
:has_field_values
:settings
:caveats
:points_of_interest
:nfc_path
:fingerprint_version
:last_analyzed
:fingerprint
:serdes/meta]
:NativeQuerySnippet [:name
:description
:entity_id
:created_at]
:Table [:name
:display_name
:description
:created_at
:db_id
:schema
:entity_type
:active
:is_upload
:field_order
:visibility_type
:show_in_getting_started
:initial_sync_status
:points_of_interest
:caveats
:serdes/meta]}
src/metabase/util.cljc
+ 0
− 6
View file @ 55ed5e0f
......@@ -841,9 +841,3 @@
[xs]
(or (empty? xs)
(apply distinct? xs)))
(defn deep-sort-map
"Converts a map (with potentially nested maps as values) into a sorted map of sorted maps."
[m]
(into (sorted-map)
(update-vals m (fn [val] (cond-> val (map? val) deep-sort-map)))))
src/metabase/util/yaml.clj
+ 5
− 9
View file @ 55ed5e0f
......@@ -25,14 +25,10 @@
(sequential? x) (mapv vectorized x)
:else x))
(defn- yamlize
"Returns x transformed for YAML output, converting dates to strings using a standard format."
[x]
(cond
(instance? Temporal x) (u.date/format x)
(map? x) (update-vals x yamlize)
(sequential? x) (map yamlize x)
:else x))
(extend-protocol yaml/YAMLCodec
Temporal
(encode [data]
(u.date/format data)))
(defn from-file
"Returns YAML parsed from file/file-like/path f, with options passed to clj-yaml."
......@@ -44,7 +40,7 @@
(defn generate-string
"Returns a YAML string from Clojure value x"
[x & {:as opts}]
(yaml/generate-string (yamlize x) opts))
(yaml/generate-string x opts))
(defn parse-string
"Returns a Clojure object parsed from YAML in string s with opts passed to clj-yaml."
......
test/metabase/util_test.cljc
+ 0
− 11
View file @ 55ed5e0f
......@@ -448,14 +448,3 @@
2 1250.04
1 1250.0
0 1250.0))
(deftest ^:parallel deep-sort-map-test
(is (= (into [] (u/deep-sort-map {:c 3 :a 1 :b 2}))
[[:a 1] [:b 2] [:c 3]])
"top level map should be sorted.")
(let [m {:d {:d.c 3 :d.a 1 :d.b 2} :c 3 :a 1 :b 2}
deeply-sorted (u/deep-sort-map m)]
(is (= (keys deeply-sorted) [:a :b :c :d]) "top level map should be sorted.")
(is (= (sort [[:d.c 3] [:d.a 1] [:d.b 2]]) (into [] (:d deeply-sorted)))
"submaps should be sorted.")))
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment