Skip to content
Snippets Groups Projects
Unverified Commit d5b25cd9 authored by github-automation-metabase's avatar github-automation-metabase Committed by GitHub
Browse files

:robot: backported "[serdes] convert database, card and dashboard with friends to using spec" (#46720) 


Co-authored-by: default avatarAlexander Solovyov <alexander@solovyov.net>
parent b1fe5fc6
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Showing
with 631 additions and 405 deletions
dev/src/dev/toucan2_monitor.clj 0 → 100644
+ 154
− 0
View file @ d5b25cd9
(ns dev.toucan2-monitor
"Utilities to track and monitor queries made by toucan2.
Usage:
(start!) ;; start tracking
;; do some query using toucan 2 or via UI
(queries) ;; get all queries and its execution time
;; => [[[\"SELECT * FROM report_card\"] 100]]
(stop!)
(to-csv!)
;; to save all queries to a csv file"
(:require
[clojure.data.csv :as csv]
[clojure.java.io :as io]
[clojure.stacktrace :as stacktrace]
[clojure.string :as str]
[dev.util :as dev.u]
[metabase.test.util.log :as tu.log]
[metabase.util :as u]
[metabase.util.log :as log]
[methodical.core :as methodical]
[toucan2.pipeline :as t2.pipeline])
(:import
(java.io File)))
(set! *warn-on-reflection* true)
(def queries*
"An atom to store all the queries and its execution time."
(atom []))
(defn queries
"Get all the queries and its execution time in ms.
Usage:
(queries)
;; => [[[\"SELECT * FROM report_card\"] 100]]"
[]
@queries*)
(defn reset-queries!
"Reset all the queries and its execution time."
[]
(reset! queries* []))
(defn summary
"Get the total number of queries and total execution time in ms."
[]
(let [qs (queries)]
{:total-queries (count qs)
:total-execution-time-ms (->> qs (map second) (apply +) int)}))
(defn call-site
"Return first callsite inside Metabase that's not `metabase.db`"
[]
(let [trace (->> (with-out-str
(stacktrace/print-stack-trace (Exception. "tracker")))
str/split-lines)]
(some-> (u/seek #(and (re-find #"^\s*metabase\." %)
(not (re-find #"^\s*metabase\.db" %))) trace)
str/trim)))
(defn- track-query-execution-fn
[next-method rf conn query-type model query]
(let [start (System/nanoTime)
result (next-method rf conn query-type model query)
end (System/nanoTime)]
(swap! queries* (fnil conj []) [query (/ (- end start) 1e6) (call-site)])
result))
(def ^:private log-thread-ref (volatile! nil))
(defn- create-log-thread! []
(Thread.
(fn []
(while (not (Thread/interrupted))
(let [{:keys [total-queries total-execution-time-ms]} (summary)]
(log/infof "Total queries: %d, Total execution time: %dms" total-queries total-execution-time-ms)
(Thread/sleep 1000))))))
(defn- start-log! []
(tu.log/set-ns-log-level! *ns* :debug)
(when-not (some? @log-thread-ref)
(let [new-thread (create-log-thread!)]
(vreset! log-thread-ref new-thread)
(.start ^Thread new-thread))))
(defn- stop-log! []
(when-let [thread @log-thread-ref]
(.interrupt ^Thread thread)
(vreset! log-thread-ref nil)))
(defn start!
"Start tracking queries."
[]
(stop-log!)
(start-log!)
(methodical/add-aux-method-with-unique-key!
#'t2.pipeline/transduce-execute-with-connection
:around
:default
track-query-execution-fn
::monitor))
(defn stop!
"Stop tracking queries."
[]
(stop-log!)
(methodical/remove-aux-method-with-unique-key!
#'t2.pipeline/transduce-execute-with-connection
:around
:default
::monitor))
(defn to-csv!
"Save all the queries and its execution time to a csv file with 3 columns: query, params, execution time."
[]
(let [qs (queries)
format-q (fn [[q t]]
[(-> q first #_mdb.query/format-sql) (-> q rest vec) t])
temp-file (File/createTempFile "queries" ".csv")]
(with-open [w (io/writer temp-file)]
(csv/write-csv w (cons ["query" "params" "execution-time(ms)"] (map format-q qs))))
(dev.u/os-open temp-file)))
(defn do-with-queries
"Implementation for [[with-queries]]."
[f]
(reset-queries!)
(start!)
(u/prog1 (f queries)
(stop!)))
(defmacro with-queries
"See Toucan queries executed:
```clj
(with-queries [queries]
(select ...)
(println :total (count (queries)))) ;; -> :total 1
```"
[[queries-binding] & body]
`(do-with-queries (^:once fn* [~queries-binding] ~@body)))
(comment
(start!)
(queries)
(stop!)
(reset-queries!)
(summary)
(to-csv!)
(doseq [q (querles)]
#_:clj-kondo/ignore
(println q)))
enterprise/backend/test/metabase_enterprise/serialization/load_test.clj
+ 0
− 12
View file @ d5b25cd9
......@@ -427,15 +427,3 @@
(let [parameters [{:values_source_config {:card_id "foo"}}]]
(with-redefs [load/fully-qualified-name->card-id {"foo" 1}]
(is (= [1] (mapv (comp :card_id :values_source_config) (#'load/resolve-dashboard-parameters parameters)))))))
(deftest with-dbs-works-as-expected-test
(ts/with-dbs [source-db dest-db]
(ts/with-db source-db
(mt/with-temp
[:model/Card _ {:name "MY CARD"}]
(testing "card is available in the source db"
(is (some? (t2/select-one :model/Card :name "MY CARD"))))
(ts/with-db dest-db
(testing "card should not be available in the dest db"
;; FAIL, select is returning a Card
(is (nil? (t2/select-one :model/Card :name "MY CARD")))))))))
enterprise/backend/test/metabase_enterprise/serialization/v2/e2e_test.clj
+ 7
− 26
View file @ d5b25cd9
......@@ -22,6 +22,7 @@
[metabase.models.setting :as setting]
[metabase.test :as mt]
[metabase.test.generate :as test-gen]
[metabase.util :as u]
[metabase.util.yaml :as yaml]
[reifyhealth.specmonstah.core :as rs]
[toucan2.core :as t2]
......@@ -54,6 +55,9 @@
(filter #(-> % :serdes/meta last :model (= model-name))
entities))
(defn- by-eid [entities entity-id]
(u/seek #(= (:entity_id %) entity-id) entities))
(defn- collections [dir]
(for [coll-dir (subdirs dir)
:when (->> ["cards" "dashboards" "timelines"]
......@@ -197,30 +201,7 @@
:collection_id [:coll 10 100]})
:timeline (many-random-fks 10 {} {:creator_id [:u 10]
:collection_id [:coll 100]})
:timeline-event (many-random-fks 90 {} {:timeline_id [:timeline 10]})
:pulse (vec (concat
;; 10 classic pulses, from collections
(many-random-fks 10 {} {:collection_id [:coll 100]})
;; 10 classic pulses, no collection
(many-random-fks 10 {:refs {:collection_id ::rs/omit}} {})
;; 10 dashboard subs
(many-random-fks 10 {:refs {:collection_id ::rs/omit}}
{:dashboard_id [:d 100]})))
:pulse-card (vec (concat
;; 60 pulse cards for the classic pulses
(many-random-fks 60 {} {:card_id [:c 100]
:pulse_id [:pulse 10]})
;; 60 pulse cards connected to dashcards for the dashboard subs
(many-random-fks 60 {} {:card_id [:c 100]
:pulse_id [:pulse 10 20]
:dashboard_card_id [:dc 300]})))
:pulse-channel (vec (concat
;; 15 channels for the classic pulses
(many-random-fks 15 {} {:pulse_id [:pulse 10]})
;; 15 channels for the dashboard subs
(many-random-fks 15 {} {:pulse_id [:pulse 10 20]})))
:pulse-channel-recipient (many-random-fks 40 {} {:pulse_channel_id [:pulse-channel 30]
:user_id [:u 100]})}))
:timeline-event (many-random-fks 90 {} {:timeline_id [:timeline 10]})}))
(is (= 101 (count (t2/select-fn-set :email 'User)))) ; +1 for the internal user
......@@ -486,7 +467,7 @@
["my-db" nil "CUSTOMERS" (:name field1s)]
nil]},
:values_source_type "card"}]
(:parameters (first (by-model extraction "Card")))))
(:parameters (by-eid extraction (:entity_id card2s)))))
(storage/store! (seq extraction) dump-dir)))
......@@ -875,7 +856,7 @@
(let [logs (mt/with-log-messages-for-level ['metabase-enterprise :error]
(let [files (->> (#'ingest/ingest-all (io/file dump-dir))
(map (comp second second))
(map #(.getName %))
(map #(.getName ^File %))
set)]
(testing "Hidden YAML wasn't read even though it's not throwing errors"
(is (not (contains? files ".hidden.yaml"))))))]
......
enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj
+ 32
− 41
View file @ d5b25cd9
......@@ -31,6 +31,7 @@
[metabase.models.action :as action]
[metabase.models.serialization :as serdes]
[metabase.test :as mt]
[metabase.util :as u]
[metabase.util.malli.schema :as ms]
[toucan2.core :as t2])
(:import
......@@ -112,20 +113,6 @@
(is (= #{coll-eid child-eid}
(by-model "Collection" (extract/extract {:user-id 218921})))))))))
(deftest database-test
(mt/with-empty-h2-app-db
(ts/with-temp-dpc [Database _ {:name "My Database"}]
(testing "without :include-database-secrets"
(let [extracted (extract/extract {})
dbs (filter #(= "Database" (:model (last (serdes/path %)))) extracted)]
(is (= 1 (count dbs)))
(is (not-any? :details dbs))))
(testing "with :include-database-secrets"
(let [extracted (extract/extract {:include-database-secrets true})
dbs (filter #(= "Database" (:model (last (serdes/path %)))) extracted)]
(is (= 1 (count dbs)))
(is (every? :details dbs)))))))
#_{:clj-kondo/ignore [:metabase/i-like-making-cams-eyes-bleed-with-horrifically-long-tests]}
(deftest dashboard-and-cards-test
(mt/with-empty-h2-app-db
......@@ -469,7 +456,7 @@
(set (serdes/dependencies ser)))))))
(testing "Dashboards include their Dashcards"
(let [ser (serdes/extract-one "Dashboard" {} (t2/select-one Dashboard :id other-dash-id))]
(let [ser (u/rfirst (serdes/extract-all "Dashboard" {:where [:= :id other-dash-id]}))]
(is (=? {:serdes/meta [{:model "Dashboard" :id other-dash :label "dave_s_dash"}]
:entity_id other-dash
:dashcards
......@@ -510,7 +497,7 @@
(set (serdes/dependencies ser)))))))
(testing "Dashboards with parameters where the source is a card"
(let [ser (serdes/extract-one "Dashboard" {} (t2/select-one Dashboard :id param-dash-id))]
(let [ser (u/rfirst (serdes/extract-all "Dashboard" {:where [:= :id param-dash-id]}))]
(is (=? {:parameters
[{:id "abc"
:name "CATEGORY"
......@@ -529,7 +516,7 @@
(set (serdes/dependencies ser))))))
(testing "Cards with parameters where the source is a card"
(let [ser (serdes/extract-one "Dashboard" {} (t2/select-one Dashboard :id param-dash-id))]
(let [ser (u/rfirst (serdes/extract-all "Dashboard" {:where [:= :id param-dash-id]}))]
(is (=? {:parameters
[{:id "abc"
:name "CATEGORY"
......@@ -582,28 +569,32 @@
(deftest dashboard-card-series-test
(mt/with-empty-h2-app-db
(ts/with-temp-dpc
[:model/Collection {coll-id :id, coll-eid :entity_id} {:name "Some Collection"}
:model/Card {c1-id :id, c1-eid :entity_id} {:name "Some Question", :collection_id coll-id}
:model/Card {c2-id :id, c2-eid :entity_id} {:name "Series Question A", :collection_id coll-id}
:model/Card {c3-id :id, c3-eid :entity_id} {:name "Series Question B", :collection_id coll-id}
:model/Dashboard {dash-id :id, dash-eid :entity_id} {:name "Shared Dashboard", :collection_id coll-id}
:model/DashboardCard {dc1-id :id, dc1-eid :entity_id} {:card_id c1-id, :dashboard_id dash-id}
:model/DashboardCard {dc2-eid :entity_id} {:card_id c1-id, :dashboard_id dash-id}
:model/DashboardCardSeries _ {:card_id c3-id, :dashboardcard_id dc1-id, :position 1}
:model/DashboardCardSeries _ {:card_id c2-id, :dashboardcard_id dc1-id, :position 0}]
(testing "Inlined dashcards include their series' card entity IDs"
(let [ser (serdes/extract-one "Dashboard" {} (t2/select-one Dashboard :id dash-id))]
(is (=? {:entity_id dash-eid
:dashcards [{:entity_id dc1-eid, :series (mt/exactly=? [{:card_id c2-eid} {:card_id c3-eid}])}
{:entity_id dc2-eid, :series []}]}
ser))
[:model/Collection {coll-id :id, coll-eid :entity_id} {:name "Some Collection"}
:model/Card {c1-id :id, c1-eid :entity_id} {:name "Some Question", :collection_id coll-id}
:model/Card {c2-id :id, c2-eid :entity_id} {:name "Series Question A", :collection_id coll-id}
:model/Card {c3-id :id, c3-eid :entity_id} {:name "Series Question B", :collection_id coll-id}
:model/Dashboard {dash-id :id, dash-eid :entity_id} {:name "Shared Dashboard", :collection_id coll-id}
:model/DashboardCard {dc1-id :id, dc1-eid :entity_id} {:card_id c1-id, :dashboard_id dash-id}
:model/DashboardCard {dc2-eid :entity_id} {:card_id c1-id, :dashboard_id dash-id}
:model/DashboardCardSeries _ {:card_id c3-id, :dashboardcard_id dc1-id, :position 1}
:model/DashboardCardSeries _ {:card_id c2-id, :dashboardcard_id dc1-id, :position 0}]
(testing "Inlined dashcards include their series' card entity IDs"
(let [ser (t2/with-call-count [q]
(u/prog1 (u/rfirst (serdes/extract-all "Dashboard" {:where [:= :id dash-id]}))
(is (< (q) 13))))]
(is (=? {:entity_id dash-eid
:dashcards [{:entity_id dc1-eid
:series (mt/exactly=? [{:card_id c2-eid :position 0}
{:card_id c3-eid :position 1}])}
{:entity_id dc2-eid, :series []}]}
ser))
(testing "and depend on all referenced cards, including cards from dashboard cards' series"
(is (= #{[{:model "Card" :id c1-eid}]
[{:model "Card" :id c2-eid}]
[{:model "Card" :id c3-eid}]
[{:model "Collection" :id coll-eid}]}
(set (serdes/dependencies ser))))))))))
(testing "and depend on all referenced cards, including cards from dashboard cards' series"
(is (= #{[{:model "Card" :id c1-eid}]
[{:model "Card" :id c2-eid}]
[{:model "Card" :id c3-eid}]
[{:model "Collection" :id coll-eid}]}
(set (serdes/dependencies ser))))))))))
(deftest dimensions-test
(mt/with-empty-h2-app-db
......@@ -635,8 +626,8 @@
:type "external"
:field_id fk-id
:human_readable_field_id cust-name}]
(testing "dimensions without foreign keys are inlined into their Fields"
(let [ser (serdes/extract-one "Field" {} (t2/select-one Field :id email-id))]
(testing "dimensions without foreign keys are inlined into their Fields\n"
(let [ser (u/rfirst (serdes/extract-all "Field" {:where [:= :id email-id]}))]
(is (malli= [:map
[:serdes/meta [:= [{:model "Database", :id "My Database"}
{:model "Table", :id "Schemaless Table"}
......@@ -658,7 +649,7 @@
(set (serdes/dependencies ser)))))))
(testing "foreign key dimensions are inlined into their Fields"
(let [ser (serdes/extract-one "Field" {} (t2/select-one Field :id fk-id))]
(let [ser (u/rfirst (serdes/extract-all "Field" {:where [:= :id fk-id]}))]
(is (malli= [:map
[:serdes/meta [:= [{:model "Database" :id "My Database"}
{:model "Schema" :id "PUBLIC"}
......
enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj
+ 130
− 7
View file @ d5b25cd9
(ns ^:mb/once metabase-enterprise.serialization.v2.load-test
(:require
[clojure.string :as str]
[clojure.test :refer :all]
[java-time.api :as t]
[metabase-enterprise.serialization.test-util :as ts]
......@@ -1093,7 +1094,7 @@
(reset! tab1s (ts/create! :model/DashboardTab :name "Tab 1" :dashboard_id (:id @dash1s)))
(reset! dashcard1s (ts/create! DashboardCard :dashboard_id (:id @dash1s) :dashboard_tab_id (:id tab1s)))
(reset! serialized (into [] (serdes.extract/extract {})))))
(reset! serialized (into [] (serdes.extract/extract {:no-settings true})))))
(testing "New dashcard will be removed on load"
(ts/with-db dest-db
......@@ -1139,7 +1140,7 @@
series1s (ts/create! :model/DashboardCardSeries :dashboardcard_id (:id dashcard1s) :card_id (:id series-card1s) :position 0)
series2s (ts/create! :model/DashboardCardSeries :dashboardcard_id (:id dashcard1s) :card_id (:id series-card2s) :position 1)
series3s (ts/create! :model/DashboardCardSeries :dashboardcard_id (:id dashcard1s) :card_id (:id series-card3s) :position 2)
extract1 (into [] (serdes.extract/extract {}))]
extract1 (into [] (serdes.extract/extract {:no-settings true}))]
(ts/with-db dest-db
(serdes.load/load-metabase! (ingestion-in-memory extract1))
(ts/with-db source-db
......@@ -1147,7 +1148,7 @@
(t2/delete! :model/DashboardCardSeries (:id series1s))
(t2/update! :model/DashboardCardSeries (:id series3s) {:position 0})
(t2/update! :model/DashboardCardSeries (:id series2s) {:position 1})
(let [extract2 (into [] (serdes.extract/extract {}))]
(let [extract2 (into [] (serdes.extract/extract {:no-settings true :no-data-model true}))]
(ts/with-db dest-db
(let [series-card2d (t2/select-one :model/Card :entity_id (:entity_id series-card2s))
series-card3d (t2/select-one :model/Card :entity_id (:entity_id series-card3s))
......@@ -1174,6 +1175,25 @@
(->> (t2/select :model/DashboardCardSeries :dashboardcard_id (:dashboardcard_id series-to-be-deleted))
(sort-by :position))))))))))))))))
(deftest dashcard-series-multi-test
(ts/with-dbs [source-db dest-db]
(testing "Dashcard series works correctly with one card in multiple series"
(ts/with-db source-db
(mt/with-temp [:model/Dashboard dash {:name "Dashboard"}
:model/Card c1 {:name "Card 1"}
:model/Card c2 {:name "Card 2"}
:model/Card sc {:name "Series Card"}
:model/DashboardCard dc1 {:card_id (:id c1) :dashboard_id (:id dash)}
:model/DashboardCard dc2 {:card_id (:id c2) :dashboard_id (:id dash)}
:model/DashboardCardSeries _s1 {:dashboardcard_id (:id dc1) :card_id (:id sc) :position 0}
:model/DashboardCardSeries _s2 {:dashboardcard_id (:id dc2) :card_id (:id sc) :position 0}]
(let [extract (into [] (serdes.extract/extract {:no-settings true}))]
(ts/with-db dest-db
(serdes.load/load-metabase! (ingestion-in-memory extract))
(testing "Both series get imported even though they point at the same card"
(is (= 2
(t2/count :model/DashboardCardSeries)))))))))))
(deftest extraneous-keys-test
(let [serialized (atom nil)
eid (u/generate-nano-id)]
......@@ -1255,12 +1275,11 @@
:visualization_settings {:click_behavior {:type "link"
:linkType "dashboard"
:targetId (:id dash1)}})
ser (atom nil)]
(reset! ser (into [] (serdes.extract/extract {:no-settings true
:no-data-model true})))
ser (into [] (serdes.extract/extract {:no-settings true
:no-data-model true}))]
(t2/delete! DashboardCard :id [:in (map :id [dc1 dc2 dc3])])
(testing "Circular dependencies are loaded correctly"
(is (serdes.load/load-metabase! (ingestion-in-memory @ser)))
(is (serdes.load/load-metabase! (ingestion-in-memory ser)))
(let [select-target #(-> % :visualization_settings :click_behavior :targetId)]
(is (= (:id dash2)
(t2/select-one-fn select-target DashboardCard :entity_id (:entity_id dc1))))
......@@ -1307,3 +1326,107 @@
(let [report (serdes.load/load-metabase! (ingestion-in-memory changed) {:continue-on-error true})]
(is (= 1 (count (:errors report))))
(is (= 3 (count (:seen report)))))))))))))))
(deftest with-dbs-works-as-expected-test
(ts/with-dbs [source-db dest-db]
(ts/with-db source-db
(mt/with-temp
[:model/Card _ {:name "MY CARD"}]
(testing "card is available in the source db"
(is (some? (t2/select-one :model/Card :name "MY CARD"))))
(ts/with-db dest-db
(testing "card should not be available in the dest db"
(is (nil? (t2/select-one :model/Card :name "MY CARD")))))))))
(deftest database-test
(ts/with-dbs [source-db dest-db]
(ts/with-db source-db
(mt/with-temp [Database _ {:name "My Database"
:details {:some "secret"}}]
(testing "without :include-database-secrets"
(let [extracted (vec (serdes.extract/extract {:no-settings true}))
dbs (filterv #(= "Database" (:model (last (serdes/path %)))) extracted)]
(is (= 1 (count dbs)))
(is (not-any? :details dbs))
(ts/with-db dest-db
(testing "loading still works even if there are no details"
(serdes.load/load-metabase! (ingestion-in-memory extracted))
(is (= {}
(t2/select-one-fn :details Database)))
(testing "If we did not export details - it won't override existing data"
(t2/update! Database {:details {:other "secret"}})
(serdes.load/load-metabase! (ingestion-in-memory extracted))
(is (= {:other "secret"}
(t2/select-one-fn :details Database)))))))))
(mt/with-temp [Database _ {:name "My Database"
:details {:some "secret"}}]
(testing "with :include-database-secrets"
(let [extracted (vec (serdes.extract/extract {:no-settings true :include-database-secrets true}))
dbs (filterv #(= "Database" (:model (last (serdes/path %)))) extracted)]
(is (= 1 (count dbs)))
(is (every? :details dbs))
(ts/with-db dest-db
(testing "Details are imported if provided"
(serdes.load/load-metabase! (ingestion-in-memory extracted))
(is (= (:details (first dbs))
(t2/select-one-fn :details Database)))))))))))
(deftest unique-dimensions-test
(ts/with-dbs [source-db dest-db]
(ts/with-db source-db
(mt/with-temp [:model/Dimension d1 {:name "Some Dimension"
:field_id (mt/id :venues :price)
:type "internal"}]
(let [ser (vec (serdes.extract/extract {:no-settings true}))]
(ts/with-db dest-db
(mt/with-temp [:model/Dimension d2 {:name "Absolutely Other Dimension"
:field_id (mt/id :venues :price)
:type "internal"}]
(serdes.load/load-metabase! (ingestion-in-memory ser))
(is (= (:entity_id d1)
(t2/select-one-fn :entity_id :model/Dimension :field_id (mt/id :venues :price))))
(is (= nil
(t2/select-one :model/Dimension :entity_id (:entity_id d2)))))))))))
(deftest nested-identity-hashes-test ;; tests serdes/nested behavior for identity hashes
(let [ids (atom {})]
(ts/with-dbs [source-db dest-db]
(ts/with-db source-db
(mt/with-temp [:model/Collection coll {:name "Coll"}
:model/Dashboard dash {:name "Dash"}
:model/Card c1 {:name "Card 1"}
:model/DashboardCard dc1 {:dashboard_id (:id dash)
:card_id (:id c1)}]
(testing "Store deserialized data ids in preparation for test"
(let [ser1 (vec (serdes.extract/extract {:no-settings true}))]
(ts/with-db dest-db
(serdes.load/load-metabase! (ingestion-in-memory ser1))
(reset! ids
(vec
(for [[_name e] {:coll coll :dash dash :c1 c1 :dc1 dc1}]
[(t2/model e) (:id (t2/select-one (t2/model e) :entity_id (:entity_id e)))]))))))
(testing "Convert everything to using identity hashes"
(t2/update! :model/Collection :id (:id coll) {:entity_id (serdes/identity-hash coll)})
(t2/update! :model/Dashboard :id (:id dash) {:entity_id (serdes/identity-hash dash)})
(t2/update! :model/Card :id (:id c1) {:entity_id (serdes/identity-hash c1)})
(t2/update! :model/DashboardCard :id (:id dc1) {:entity_id (serdes/identity-hash dc1)}))
(is (= 8 (count (serdes/entity-id "Card"
(t2/select-one [:model/Card :entity_id] :id (:id c1))))))
(testing "Identity hashes end up in target db in place of entity ids"
(let [ser2 (vec (serdes.extract/extract {:no-settings true :no-data-model true}))]
(testing "\nWe exported identity hashes"
(doseq [e ser2
:when (:entity_id e)]
(is (= 8 (count (-> e :entity_id str/trim))))))
(ts/with-db dest-db
(serdes.load/load-metabase! (ingestion-in-memory ser2))
(testing "\nAll entities (including nested dashcards) were updated"
(doseq [[model id] @ids
:let [e (t2/select-one model :id id)]]
(testing (format "%s has identity hash in the db" model)
(is (= (serdes/identity-hash e)
(serdes/entity-id (name model) e))))))))))))))
enterprise/backend/test/metabase_enterprise/serialization/v2/models_test.clj
+ 29
− 26
View file @ d5b25cd9
(ns metabase-enterprise.serialization.v2.models-test
(:require
[clojure.set :as set]
[clojure.test :refer :all]
[metabase-enterprise.serialization.v2.backfill-ids :as serdes.backfill]
[metabase-enterprise.serialization.v2.entity-ids :as v2.entity-ids]
......@@ -44,34 +45,36 @@
(deftest serialization-complete-spec-test
(mt/with-empty-h2-app-db
;; When serialization spec is defined, it describes every column
(doseq [m serdes.models/exported-models
:let [spec (serdes/make-spec m)]
:when spec]
(doseq [m (-> (methods serdes/make-spec)
(dissoc :default)
keys)
:let [spec (serdes/make-spec m nil)]]
(let [t (t2/table-name (keyword "model" m))
fields (u.conn/app-db-column-types (mdb/app-db) t)
spec' (merge (zipmap (:copy spec) (repeat :copy))
(zipmap (:skip spec) (repeat :skip))
(:transform spec))]
(testing (format "%s should declare every column in serialization spec" m)
(is (= (->> (keys fields)
(map u/lower-case-en)
set)
(->> (keys spec')
(map name)
set))))
spec' (-> (merge (zipmap (:copy spec) (repeat :copy))
(zipmap (:skip spec) (repeat :skip))
(zipmap [:id :updated_at] (repeat :skip)) ; always skipped
(:transform spec))
;; `nil`s are mostly fields which differ on `opts`
(dissoc nil))]
(testing (format "%s has no duplicates in serialization spec\n" m)
(are [x y] (empty? (set/intersection (set x) (set y)))
(:copy spec) (:skip spec)
(:copy spec) (keys (:transform spec))
(:skip spec) (keys (:transform spec))))
(testing (format "%s should declare every column in serialization spec\n" m)
(is (set/subset?
(->> (keys fields)
(map u/lower-case-en)
set)
(->> (keys spec')
(map name)
set))))
(testing "Foreign keys should be declared as such\n"
(doseq [[fk _] (filter #(:fk (second %)) fields)
:let [fk (u/lower-case-en fk)
action (get spec' (keyword fk))]]
:let [fk (u/lower-case-en fk)
transform (get spec' (keyword fk))]
:when (not= transform :skip)]
(testing (format "%s.%s is foreign key which is handled correctly" m fk)
;; FIXME: serialization can guess where FK points by itself, but `collection_id` and `database_id` are
;; specifying that themselves right now
(when-not (#{"collection_id" "database_id"} fk)
(is (#{:skip
serdes/*export-fk*
serdes/*export-fk-keyed*
serdes/*export-table-fk*
serdes/*export-user*}
(if (vector? action)
(first action) ;; tuple of [ser des]
action)))))))))))
;; uses `(serdes/fk ...)` function
(is (::serdes/fk transform)))))))))
src/metabase/models/card.clj
+ 14
− 25
View file @ d5b25cd9
......@@ -1023,38 +1023,27 @@ saved later when it is ready."
(when (:id m) #{(serdes/field->path (:id m))})])))))
(defmethod serdes/make-spec "Card"
[_model-name]
{:copy [:archived :collection_position :collection_preview :created_at :description :display
[_model-name _opts]
{:copy [:archived :archived_directly :collection_position :collection_preview :created_at :description :display
:embedding_params :enable_embedding :entity_id :metabase_version :public_uuid :query_type :type :name]
:skip [ ;; always instance-specific
:id :updated_at
;; cache invalidation is instance-specific
:skip [;; cache invalidation is instance-specific
:cache_invalidated_at
;; those are instance-specific analytic columns
:view_count :last_used_at :initially_published_at
;; this column is not used anymore
:cache_ttl]
:transform
{:database_id [#(serdes/*export-fk-keyed* % 'Database :name)
#(serdes/*import-fk-keyed* % 'Database :name)]
:table_id [serdes/*export-table-fk*
serdes/*import-table-fk*]
:collection_id [#(serdes/*export-fk* % 'Collection)
#(serdes/*import-fk* % 'Collection)]
:creator_id [serdes/*export-user*
serdes/*import-user*]
:made_public_by_id [serdes/*export-user*
serdes/*import-user*]
:dataset_query [serdes/export-mbql
serdes/import-mbql]
:parameters [serdes/export-parameters
serdes/import-parameters]
:parameter_mappings [serdes/export-parameter-mappings
serdes/import-parameter-mappings]
:visualization_settings [serdes/export-visualization-settings
serdes/import-visualization-settings]
:result_metadata [export-result-metadata
import-result-metadata]}})
{:database_id (serdes/fk :model/Database :name)
:table_id (serdes/fk :model/Table)
:source_card_id (serdes/fk :model/Card)
:collection_id (serdes/fk :model/Collection)
:creator_id (serdes/fk :model/User)
:made_public_by_id (serdes/fk :model/User)
:dataset_query {:export serdes/export-mbql :import serdes/import-mbql}
:parameters {:export serdes/export-parameters :import serdes/import-parameters}
:parameter_mappings {:export serdes/export-parameter-mappings :import serdes/import-parameter-mappings}
:visualization_settings {:export serdes/export-visualization-settings :import serdes/import-visualization-settings}
:result_metadata {:export export-result-metadata :import import-result-metadata}}})
(defmethod serdes/dependencies "Card"
[{:keys [collection_id database_id dataset_query parameters parameter_mappings
......
src/metabase/models/dashboard.clj
+ 22
− 95
View file @ d5b25cd9
......@@ -578,107 +578,34 @@
;;; +----------------------------------------------------------------------------------------------------------------+
;;; | SERIALIZATION |
;;; +----------------------------------------------------------------------------------------------------------------+
(defmethod serdes/extract-query "Dashboard" [_ opts]
(eduction (map #(t2/hydrate % [:dashcards :series]))
(eduction (map #(t2/hydrate % :tabs [:dashcards :series]))
(serdes/extract-query-collections Dashboard opts)))
(defn export-dashboard-card-series
"Given the hydrated `:series` of a DashboardCard, as a vector of maps, converts it to a portable form with
the card IDs replaced with their entity IDs."
[cards]
(mapv (fn [card]
{:card_id (serdes/*export-fk* (:id card) :model/Card)})
cards))
(defn- extract-dashcard
[dashcard]
(-> (into (sorted-map) dashcard)
(dissoc :id :collection_authority_level :dashboard_id :updated_at)
(update :card_id serdes/*export-fk* 'Card)
(update :action_id serdes/*export-fk* 'Action)
(update :dashboard_tab_id serdes/*export-fk* :model/DashboardTab)
(update :series export-dashboard-card-series)
(update :parameter_mappings serdes/export-parameter-mappings)
(update :visualization_settings serdes/export-visualization-settings)))
(defn- extract-dashtab
[dashtab]
(dissoc dashtab :id :dashboard_id :updated_at))
(defmethod serdes/extract-one "Dashboard"
[_model-name _opts dash]
(let [dash (cond-> dash
(nil? (:dashcards dash))
(t2/hydrate [:dashcards :series])
(nil? (:tabs dash))
(t2/hydrate :tabs))]
(-> (serdes/extract-one-basics "Dashboard" dash)
(update :dashcards #(mapv extract-dashcard %))
(update :tabs #(mapv extract-dashtab %))
(update :parameters serdes/export-parameters)
(update :collection_id serdes/*export-fk* Collection)
(update :creator_id serdes/*export-user*)
(update :made_public_by_id serdes/*export-user*)
(dissoc :view_count))))
(defmethod serdes/load-xform "Dashboard"
[dash]
(-> dash
serdes/load-xform-basics
;; Deliberately not doing anything to :dashcards - they get handled by load-one! below.
(update :collection_id serdes/*import-fk* Collection)
(update :parameters serdes/import-parameters)
(update :creator_id serdes/*import-user*)
(update :made_public_by_id serdes/*import-user*)))
(defn- dashcard-for [dashcard dashboard]
(assoc dashcard
:dashboard_id (:entity_id dashboard)
:serdes/meta (remove nil?
[{:model "Dashboard" :id (:entity_id dashboard)}
(when-let [dashtab-eeid (last (:dashboard_tab_id dashcard))]
{:model "DashboardTab" :id dashtab-eeid})
{:model "DashboardCard" :id (:entity_id dashcard)}])))
(defn- dashtab-for [tab dashboard]
(assoc tab
:dashboard_id (:entity_id dashboard)
:serdes/meta [{:model "Dashboard" :id (:entity_id dashboard)}
{:model "DashboardTab" :id (:entity_id tab)}]))
(defn- drop-excessive-nested!
"Remove nested entities which are not present in incoming serialization load"
[hydration-key ingested local]
(let [local-nested (get (t2/hydrate local hydration-key) hydration-key)
ingested-nested (get ingested hydration-key)
to-remove (set/difference (set (map :entity_id local-nested))
(set (map :entity_id ingested-nested)))
model (t2/model (first local-nested))]
(when (seq to-remove)
(t2/delete! model :entity_id [:in to-remove]))))
;; Call the default load-one! for the Dashboard, then for each DashboardCard.
(defmethod serdes/load-one! "Dashboard" [ingested maybe-local]
(let [dashboard ((get-method serdes/load-one! :default) (dissoc ingested :dashcards :tabs) maybe-local)]
(drop-excessive-nested! :tabs ingested dashboard)
(doseq [tab (:tabs ingested)]
(serdes/load-one! (dashtab-for tab dashboard)
(t2/select-one :model/DashboardTab :entity_id (:entity_id tab))))
(drop-excessive-nested! :dashcards ingested dashboard)
(doseq [dashcard (:dashcards ingested)]
(serdes/load-one! (dashcard-for dashcard dashboard)
(t2/select-one :model/DashboardCard :entity_id (:entity_id dashcard))))))
(defmethod serdes/make-spec "Dashboard" [_model-name opts]
{:copy [:archived :archived_directly :auto_apply_filters :cache_ttl :caveats :collection_position
:description :embedding_params :enable_embedding :entity_id :initially_published_at :name
:points_of_interest :position :public_uuid :show_in_getting_started :width]
:skip [;; those stats are inherently local state
:view_count :last_viewed_at]
:transform {:created_at (serdes/date)
:collection_id (serdes/fk :model/Collection)
:creator_id (serdes/fk :model/User)
:made_public_by_id (serdes/fk :model/User)
:parameters {:export serdes/export-parameters :import serdes/import-parameters}
:tabs (serdes/nested :model/DashboardTab :dashboard_id opts)
:dashcards (serdes/nested :model/DashboardCard :dashboard_id opts)}})
(defn- serdes-deps-dashcard
[{:keys [action_id card_id parameter_mappings visualization_settings series]}]
(->> (mapcat serdes/mbql-deps parameter_mappings)
(concat (serdes/visualization-settings-deps visualization_settings))
(concat (when card_id #{[{:model "Card" :id card_id}]}))
(concat (when action_id #{[{:model "Action" :id action_id}]}))
(concat (for [s series] [{:model "Card" :id (:card_id s)}]))
set))
(set
(concat
(mapcat serdes/mbql-deps parameter_mappings)
(serdes/visualization-settings-deps visualization_settings)
(when card_id #{[{:model "Card" :id card_id}]})
(when action_id #{[{:model "Action" :id action_id}]})
(for [s series] [{:model "Card" :id (:card_id s)}]))))
(defmethod serdes/dependencies "Dashboard"
[{:keys [collection_id dashcards parameters]}]
......
src/metabase/models/dashboard_card.clj
+ 25
− 36
View file @ d5b25cd9
......@@ -9,7 +9,6 @@
[metabase.models.pulse-card :refer [PulseCard]]
[metabase.models.serialization :as serdes]
[metabase.util :as u]
[metabase.util.date-2 :as u.date]
[metabase.util.honey-sql-2 :as h2x]
[metabase.util.malli :as mu]
[metabase.util.malli.schema :as ms]
......@@ -353,9 +352,7 @@
(compare row-1 row-2)))
;;; ----------------------------------------------- SERIALIZATION ----------------------------------------------------
;; DashboardCards are not serialized as their own, separate entities. They are inlined onto their parent Dashboards.
;; If the parent dashboard has tabs, the dashcards are inlined under each DashboardTab, which are inlined on the Dashboard.
;; However, we can reuse some of the serdes machinery (especially load-one!) by implementing a few serdes methods.
(defmethod serdes/generate-path "DashboardCard" [_ dashcard]
(remove nil?
[(serdes/infer-self-path "Dashboard" (t2/select-one 'Dashboard :id (:dashboard_id dashcard)))
......@@ -363,35 +360,27 @@
(serdes/infer-self-path "DashboardTab" (t2/select-one :model/DashboardTab :id (:dashboard_tab_id dashcard))))
(serdes/infer-self-path "DashboardCard" dashcard)]))
(defmethod serdes/load-xform "DashboardCard"
[dashcard]
(-> dashcard
;; Deliberately not doing anything to :series, they get handled by load-one! below
(dissoc :serdes/meta)
(update :card_id serdes/*import-fk* :model/Card)
(update :action_id serdes/*import-fk* :model/Action)
(update :dashboard_id serdes/*import-fk* :model/Dashboard)
(update :dashboard_tab_id serdes/*import-fk* :model/DashboardTab)
(update :created_at #(if (string? %) (u.date/parse %) %))
(update :parameter_mappings serdes/import-parameter-mappings)
(update :visualization_settings serdes/import-visualization-settings)))
(defn- dashboard-card-series-xform
[ingested]
(-> ingested
(update :card_id serdes/*import-fk* :model/Card)
(update :dashboardcard_id serdes/*import-fk* :model/DashboardCard)))
(defmethod serdes/load-one! "DashboardCard"
[ingested maybe-local]
(let [dashcard ((get-method serdes/load-one! :default) (dissoc ingested :series) maybe-local)]
;; drop all existing series for this card and recreate them
;; TODO: this is unnecessary, but it is simple to implement
(t2/delete! :model/DashboardCardSeries :dashboardcard_id (:id dashcard))
(doseq [[idx single-series] (map-indexed vector (:series ingested))] ;; a single series has a :card_id only
;; instead of load-one! we use load-insert! here because :serdes/meta isn't necessary because no other
;; entities depend on DashboardCardSeries
(serdes/load-insert! "DashboardCardSeries" (-> single-series
(assoc :dashboardcard_id (:entity_id dashcard)
:position idx)
dashboard-card-series-xform)))))
(defmethod serdes/make-spec "DashboardCard" [_model-name opts]
{:copy [:col :entity_id :row :size_x :size_y]
:skip []
:transform {:created_at (serdes/date)
:dashboard_id (serdes/parent-ref)
:card_id (serdes/fk :model/Card)
:action_id (serdes/fk :model/Action)
:dashboard_tab_id (serdes/fk :model/DashboardTab)
:parameter_mappings {:export serdes/export-parameter-mappings
:import serdes/import-parameter-mappings}
:visualization_settings {:export serdes/export-visualization-settings
:import serdes/import-visualization-settings}
:series
(-> (serdes/nested :model/DashboardCardSeries :dashboardcard_id
(assoc opts
:sort-by :position
:key-field :card_id))
;; FIXME: this waits to be removed when `extract-nested` (instead of using hydration) is
;; implemented; see comment at `make-spec` for `DashboardCardSeries`
(assoc :export (fn [data]
(vec (map-indexed (fn [i x]
{:card_id (serdes/*export-fk* (:id x) :model/Card)
:position i})
data)))))}})
src/metabase/models/dashboard_card_series.clj
+ 15
− 0
View file @ d5b25cd9
(ns metabase.models.dashboard-card-series
(:require
[metabase.models.serialization :as serdes]
[methodical.core :as methodical]
[toucan2.core :as t2]))
......@@ -12,3 +13,17 @@
(doto :model/DashboardCardSeries
(derive :metabase/model))
;; Serialization
(defmethod serdes/generate-path "DashboardCardSeries" [_ _] nil)
;; TODO: this is not used atm as `DashboardCard` has custom :export/:import defined; see comment there
;; to be implemented.
(defmethod serdes/make-spec "DashboardCardSeries" [_model-name _opts]
;; We did not have `position` in serialization before, it was inferred from the order, but we're trying to keep
;; code more generic right now - so it's carried over as data rather than implied.
{:copy [:position]
:skip []
:transform {:dashboardcard_id (serdes/parent-ref)
:card_id (serdes/fk :model/Card)}})
src/metabase/models/dashboard_tab.clj
+ 5
− 8
View file @ d5b25cd9
......@@ -5,7 +5,6 @@
[metabase.models.interface :as mi]
[metabase.models.serialization :as serdes]
[metabase.util :as u]
[metabase.util.date-2 :as u.date]
[metabase.util.malli :as mu]
[metabase.util.malli.schema :as ms]
[methodical.core :as methodical]
......@@ -59,17 +58,15 @@
:position
:created_at])
;; DashboardTabs are not serialized as their own, separate entities. They are inlined onto their parent Dashboards.
(defmethod serdes/generate-path "DashboardTab" [_ dashcard]
[(serdes/infer-self-path "Dashboard" (t2/select-one :model/Dashboard :id (:dashboard_id dashcard)))
(serdes/infer-self-path "DashboardTab" dashcard)])
(defmethod serdes/load-xform "DashboardTab"
[dashtab]
(-> dashtab
(dissoc :serdes/meta)
(update :dashboard_id serdes/*import-fk* :model/Dashboard)
(update :created_at #(if (string? %) (u.date/parse %) %))))
(defmethod serdes/make-spec "DashboardTab" [_model-name _opts]
{:copy [:entity_id :name :position]
:skip []
:transform {:created_at (serdes/date)
:dashboard_id (serdes/parent-ref)}})
;;; -------------------------------------------------- CRUD fns ------------------------------------------------------
......
src/metabase/models/database.clj
+ 11
− 26
View file @ d5b25cd9
......@@ -432,12 +432,17 @@
;;; ------------------------------------------------ Serialization ----------------------------------------------------
(defmethod serdes/extract-one "Database"
[_model-name {:keys [include-database-secrets]} entity]
(-> (serdes/extract-one-basics "Database" entity)
(update :creator_id serdes/*export-user*)
(dissoc :features) ; This is a synthetic column that isn't in the real schema.
(cond-> (not include-database-secrets) (dissoc :details))))
(defmethod serdes/make-spec "Database"
[_model-name {:keys [include-database-secrets]}]
{:copy [:auto_run_queries :cache_field_values_schedule :cache_ttl :caveats :created_at :dbms_version
:description :engine :is_audit :is_full_sync :is_on_demand :is_sample :metadata_sync_schedule :name
:points_of_interest :refingerprint :settings :timezone :uploads_enabled :uploads_schema_name
:uploads_table_prefix]
:skip []
:transform {;; details should be imported if available regardless of options
:details {:export #(when include-database-secrets %) :import identity}
:creator_id (serdes/fk :model/User)
:initial_sync_status {:export identity :import (constantly "complete")}}})
(defmethod serdes/entity-id "Database"
[_ {:keys [name]}]
......@@ -451,26 +456,6 @@
[[{:keys [id]}]]
(t2/select-one Database :name id))
(defmethod serdes/load-xform "Database"
[database]
(-> database
serdes/load-xform-basics
(update :creator_id serdes/*import-user*)
(assoc :initial_sync_status "complete")))
(defmethod serdes/load-insert! "Database" [_ ingested]
(let [m (get-method serdes/load-insert! :default)]
(m "Database"
(if (:details ingested)
ingested
(assoc ingested :details {})))))
(defmethod serdes/load-update! "Database" [_ ingested local]
(let [m (get-method serdes/load-update! :default)]
(m "Database"
(update ingested :details #(or % (:details local) {}))
local)))
(defmethod serdes/storage-path "Database" [{:keys [name]} _]
;; ["databases" "db_name" "db_name"] directory for the database with same-named file inside.
["databases" name name])
......
src/metabase/models/dimension.clj
+ 7
− 10
View file @ d5b25cd9
......@@ -5,7 +5,6 @@
(:require
[metabase.models.interface :as mi]
[metabase.models.serialization :as serdes]
[metabase.util.date-2 :as u.date]
[methodical.core :as methodical]
[toucan2.core :as t2]))
......@@ -36,12 +35,10 @@
:created_at])
;;; ------------------------------------------------- Serialization --------------------------------------------------
;; Dimensions are inlined onto their parent Fields.
;; We can reuse the [[serdes/load-one!]] logic by implementing [[serdes/load-xform]] though.
(defmethod serdes/load-xform "Dimension"
[dim]
(-> dim
serdes/load-xform-basics
;; No need to handle :field_id, it was just added as the raw ID by the caller; see Field's load-one!
(update :human_readable_field_id serdes/*import-field-fk*)
(update :created_at u.date/parse)))
(defmethod serdes/make-spec "Dimension" [_model-name _opts]
{:copy [:name :type :entity_id]
:skip []
:transform {:created_at (serdes/date)
:human_readable_field_id (serdes/fk :model/Field)
:field_id (serdes/parent-ref)}})
src/metabase/models/field.clj
+ 21
− 44
View file @ d5b25cd9
......@@ -372,12 +372,17 @@
(defmethod serdes/entity-id "Field" [_ {:keys [name]}]
name)
(defmethod serdes/extract-query "Field" [_model-name _opts]
(let [d (t2/select Dimension)
(defmethod serdes/load-find-local "Field"
[path]
(let [table (serdes/load-find-local (pop path))]
(t2/select-one Field :name (-> path last :id) :table_id (:id table))))
(defmethod serdes/extract-query "Field" [_model-name opts]
(let [d (t2/select Dimension)
dimensions (->> d
(group-by :field_id))]
(eduction (map #(assoc % :dimensions (get dimensions (:id %))))
(t2/reducible-select Field))))
(t2/reducible-select Field {:where (:where opts true)}))))
(defmethod serdes/dependencies "Field" [field]
;; Fields depend on their parent Table, plus any foreign Fields referenced by their Dimensions.
......@@ -393,49 +398,21 @@
fks (set/union #{fks})
true (disj this))))
(defn- extract-dimensions [dimensions]
(->> (for [dim dimensions]
(-> (into (sorted-map) dim)
(dissoc :field_id :updated_at) ; :field_id is implied by the nesting under that field.
(update :human_readable_field_id serdes/*export-field-fk*)))
(sort-by :created_at)))
(defmethod serdes/extract-one "Field"
[_model-name _opts field]
(let [field (if (contains? field :dimensions)
field
(assoc field :dimensions (t2/select Dimension :field_id (:id field))))]
(-> (serdes/extract-one-basics "Field" field)
(update :dimensions extract-dimensions)
(update :table_id serdes/*export-table-fk*)
(update :fk_target_field_id serdes/*export-field-fk*)
(update :parent_id serdes/*export-field-fk*)
(dissoc :fingerprint :last_analyzed :fingerprint_version))))
(defmethod serdes/load-xform "Field"
[field]
(-> (serdes/load-xform-basics field)
(update :table_id serdes/*import-table-fk*)
(update :fk_target_field_id serdes/*import-field-fk*)
(update :parent_id serdes/*import-field-fk*)))
(defmethod serdes/load-find-local "Field"
[path]
(let [table (serdes/load-find-local (pop path))]
(t2/select-one Field :name (-> path last :id) :table_id (:id table))))
(defmethod serdes/load-one! "Field" [ingested maybe-local]
(let [field ((get-method serdes/load-one! :default) (dissoc ingested :dimensions) maybe-local)]
(doseq [dim (:dimensions ingested)]
(let [local (t2/select-one Dimension :entity_id (:entity_id dim))
dim (assoc dim
:field_id (:id field)
:serdes/meta [{:model "Dimension" :id (:entity_id dim)}])]
(serdes/load-one! dim local)))))
(defmethod serdes/make-spec "Field" [_model-name opts]
{:copy [:active :base_type :caveats :coercion_strategy :custom_position :database_indexed
:database_is_auto_increment :database_partitioned :database_position :database_required :database_type
:description :display_name :effective_type :has_field_values :is_defective_duplicate :json_unfolding
:name :nfc_path :points_of_interest :position :preview_display :semantic_type :settings
:unique_field_helper :visibility_type]
:skip [:fingerprint :fingerprint_version :last_analyzed]
:transform {:created_at (serdes/date)
:table_id (serdes/fk :model/Table)
:fk_target_field_id (serdes/fk :model/Field)
:parent_id (serdes/fk :model/Field)
:dimensions (serdes/nested :model/Dimension :field_id opts)}})
(defmethod serdes/storage-path "Field" [field _]
(-> field
serdes/path
(-> (serdes/path field)
drop-last
serdes/storage-table-path-prefix
(concat ["fields" (:name field)])))
src/metabase/models/field_values.clj
+ 3
− 0
View file @ d5b25cd9
......@@ -559,6 +559,9 @@
;;; +----------------------------------------------------------------------------------------------------------------+
;;; | Serialization |
;;; +----------------------------------------------------------------------------------------------------------------+
(defmethod serdes/entity-id "FieldValues" [_ _] nil)
(defmethod serdes/generate-path "FieldValues" [_ {:keys [field_id]}]
(let [field (t2/select-one 'Field :id field_id)]
(conj (serdes/generate-path "Field" field)
......
src/metabase/models/serialization.clj
+ 149
− 48
View file @ d5b25cd9
......@@ -24,9 +24,11 @@
[metabase.shared.models.visualization-settings :as mb.viz]
[metabase.util :as u]
[metabase.util.connection :as u.conn]
[metabase.util.date-2 :as u.date]
[metabase.util.log :as log]
[toucan2.core :as t2]
[toucan2.model :as t2.model]))
[toucan2.model :as t2.model]
[toucan2.realize :as t2.realize]))
(set! *warn-on-reflection* true)
......@@ -69,6 +71,8 @@
;;; - For entities that existed before the column was added, have a portable way to rebuild them (see below on
;;; hashing).
(def ^:private ^:dynamic *current* "Instance/map being exported/imported currently" nil)
(defmulti entity-id
"Given the model name and an entity, returns its entity ID (which might be nil).
......@@ -79,8 +83,8 @@
{:arglists '([model-name instance])}
(fn [model-name _instance] model-name))
(defmethod entity-id :default [_ {:keys [entity_id]}]
(str/trim entity_id))
(defmethod entity-id :default [_ instance]
(some-> instance :entity_id str/trim))
(defn eid->id
"Given model name and its entity id, returns it database-local id.
......@@ -193,7 +197,9 @@
`(generate-path \"ModelName\" entity)`
The path is a vector of maps, root first and this entity itself last. Each map looks like:
`{:model \"ModelName\" :id \"entity ID, identity hash, or custom ID\" :label \"optional human label\"}`"
`{:model \"ModelName\" :id \"entity ID, identity hash, or custom ID\" :label \"optional human label\"}`
Nested models with no entity_id need to return nil for generate-path."
{:arglists '([model-name instance])}
(fn [model-name _instance] model-name))
......@@ -214,13 +220,12 @@
For example, a Card's or Dashboard's `:name` field."
[model-name entity slug-key]
(let [self (infer-self-path model-name entity)
label (get entity slug-key)]
[(if label
(assoc self :label (u/slugify label {:unicode? true}))
self)]))
label (slug-key entity)]
[(-> self
(m/assoc-some :label (some-> label (u/slugify {:unicode? true}))))]))
(defmethod generate-path :default [model-name entity]
;; This default works for most models, but needs overriding for nested ones.
;; This default works for most models, but needs overriding for those that don't rely on entity_id.
(maybe-labeled model-name entity :name))
;;; # Export Process
......@@ -311,16 +316,27 @@
`:transform` is a map from field name to a `{:ser (fn [v] ...) :des (fn [v] ...)}` map with functions to
serialize/deserialize data.
For behavior, see `extract-by-spec` and `load-by-spec`."
(fn [model-name] model-name))
For behavior, see `extract-by-spec` and `xform-by-spec`."
(fn [model-name _opts] model-name))
(defmethod make-spec :default [_] nil)
(defmethod make-spec :default [_ _] nil)
(defn- extract-by-spec [model-name _opts instance]
(when-let [spec (make-spec model-name)]
(into (select-keys instance (:copy spec))
(for [[k [ser _des]] (:transform spec)]
[k (ser (get instance k))]))))
(defn- extract-by-spec [model-name opts instance]
(try
(binding [*current* instance]
(when-let [spec (make-spec model-name opts)]
(-> (select-keys instance (:copy spec))
;; won't assoc if `generate-path` returned `nil`
(m/assoc-some :serdes/meta (generate-path model-name instance))
(into (for [[k transform] (:transform spec)
:let [res ((:export transform) (get instance k))]
;; include only non-nil transform results
:when res]
[k res])))))
(catch Exception e
(throw (ex-info (format "Error extracting %s %s" model-name (:id instance))
(assoc (ex-data e) :model model-name :id (:id instance))
e)))))
(defmulti extract-all
"Entry point for extracting all entities of a particular model:
......@@ -372,7 +388,7 @@
(defn log-and-extract-one
"Extracts a single entity; will replace `extract-one` as public interface once `extract-one` overrides are gone."
[model opts instance]
(log/infof "Extracting %s %s" model (:id instance))
(log/infof "Extracting %s %s %s" model (:id instance) (entity-id model instance))
(try
(extract-one model opts instance)
(catch Exception e
......@@ -396,18 +412,20 @@
(defn extract-query-collections
"Helper for the common (but not default) [[extract-query]] case of fetching everything that isn't in a personal
collection."
[model {:keys [collection-set]}]
[model {:keys [collection-set where]}]
(if collection-set
;; If collection-set is defined, select everything in those collections, or with nil :collection_id.
(let [in-colls (t2/reducible-select model :collection_id [:in collection-set])]
(if (contains? collection-set nil)
(eduction cat [in-colls (t2/reducible-select model :collection_id nil)])
in-colls))
(t2/reducible-select model {:where [:or
[:in :collection_id collection-set]
(when (contains? collection-set nil)
[:= :collection_id nil])
(when where
where)]})
;; If collection-set is nil, just select everything.
(t2/reducible-select model)))
(t2/reducible-select model {:where (or where true)})))
(defmethod extract-query :default [model-name _]
(t2/reducible-select (symbol model-name)))
(defmethod extract-query :default [model-name {:keys [where]}]
(t2/reducible-select (symbol model-name) {:where (or where true)}))
(defn extract-one-basics
"A helper for writing [[extract-one]] implementations. It takes care of the basics:
......@@ -427,8 +445,7 @@
(defmethod extract-one :default [model-name opts entity]
;; `extract-by-spec` is called here since most of tests use `extract-one` right now
(or (some-> (extract-by-spec model-name opts entity)
(assoc :serdes/meta (generate-path model-name entity)))
(or (extract-by-spec model-name opts entity)
(extract-one-basics model-name entity)))
(defmulti descendants
......@@ -680,24 +697,36 @@
(fn [ingested _]
(ingested-model ingested)))
(defn- load-by-spec [ingested]
(let [model-name (ingested-model ingested)
spec (make-spec model-name)]
(defn- xform-by-spec [model-name ingested]
(let [spec (make-spec model-name nil)]
(when spec
(into (select-keys ingested (:copy spec))
(for [[k [_ser des]] (:transform spec)]
[k (des (get ingested k))])))))
(-> (select-keys ingested (:copy spec))
(into (for [[k transform] (:transform spec)
:when (not (::nested transform))
:let [res ((:import transform) (get ingested k))]
;; do not try to insert nil values if transformer returns nothing
:when res]
[k res]))))))
(defn- spec-nested! [model-name ingested instance]
(binding [*current* instance]
(let [spec (make-spec model-name nil)]
(doseq [[k transform] (:transform spec)
:when (::nested transform)]
((:import transform) (get ingested k))))))
(defn default-load-one!
"Default implementation of `load-one!`"
[ingested maybe-local]
(let [model (ingested-model ingested)
adjusted (or (load-by-spec ingested)
(load-xform ingested))]
(binding [mi/*deserializing?* true]
(if (nil? maybe-local)
(load-insert! model adjusted)
(load-update! model adjusted maybe-local)))))
(let [model-name (ingested-model ingested)
adjusted (or (xform-by-spec model-name ingested)
(load-xform ingested))
instance (binding [mi/*deserializing?* true]
(if (nil? maybe-local)
(load-insert! model-name adjusted)
(load-update! model-name adjusted maybe-local)))]
(spec-nested! model-name ingested instance)
instance))
(defmethod load-one! :default [ingested maybe-local]
(default-load-one! ingested maybe-local))
......@@ -719,6 +748,7 @@
[model id-hash]
(->> (t2/reducible-select model)
(into [] (comp (filter #(= id-hash (identity-hash %)))
(map t2.realize/realize)
(take 1)))
first))
......@@ -800,7 +830,6 @@
[id model]
(when id
(let [model-name (name model)
model (t2.model/resolve-model (symbol model-name))
entity (t2/select-one model (first (t2/primary-keys model)) id)
path (when entity
(mapv :id (generate-path model-name entity)))]
......@@ -824,12 +853,10 @@
Unusual parameter order means this can be used as `(update x :some_id import-fk 'SomeModel)`."
[eid model]
(when eid
(let [model-name (name model)
model (t2.model/resolve-model (symbol model-name))
eid (if (vector? eid)
(last eid)
eid)
entity (lookup-by-id model eid)]
(let [eid (if (vector? eid)
(last eid)
eid)
entity (lookup-by-id model eid)]
(if entity
(get entity (first (t2/primary-keys model)))
(throw (ex-info "Could not find foreign key target - bad serdes dependencies or other serialization error"
......@@ -1486,6 +1513,80 @@
(set/union (viz-click-behavior-descendants viz)
(viz-column-settings-descendants viz)))
;;; Common transformers
(defn fk "Export Foreign Key" [model & [field-name]]
(cond
;; this `::fk` is used in tests to determine that foreign keys are handled
(= model :model/User) {::fk true :export *export-user* :import *import-user*}
(= model :model/Table) {::fk true :export *export-table-fk* :import *import-table-fk*}
(= model :model/Field) {::fk true :export *export-field-fk* :import *import-field-fk*}
field-name {::fk true
:export #(*export-fk-keyed* % model field-name)
:import #(*import-fk-keyed* % model field-name)}
:else {::fk true :export #(*export-fk* % model) :import #(*import-fk* % model)}))
(defn nested "Nested entities" [model backward-fk opts]
(let [model-name (name model)
sorter (:sort-by opts :created_at)
key-field (:key-field opts :entity_id)]
{::nested true
:model model
:backward-fk backward-fk
:export (fn [data]
(assert (every? #(t2/instance-of? model %) data)
(format "Nested data is expected to be a %s, not %s" model (t2/model (first data))))
;; `nil? data` check is for `extract-one` case in tests; make sure to add empty vectors in
;; `extract-query` implementations for nested collections
(try
(->> (or data (when (nil? data)
(t2/select model backward-fk (:id *current*))))
(sort-by sorter)
(mapv #(extract-one model-name opts %)))
(catch Exception e
(throw (ex-info (format "Error exporting nested %s" model)
{:model model
:parent-id (:id *current*)}
e)))))
:import (fn [lst]
(let [parent-id (:id *current*)
first-eid (some->> (first lst)
(entity-id model-name))
enrich (fn [ingested]
(-> ingested
(assoc backward-fk parent-id)
(update :serdes/meta #(or % [{:model model-name :id (get ingested key-field)}]))))]
(cond
(nil? first-eid) ; no entity id, just drop existing stuff
(do (t2/delete! model backward-fk parent-id)
(doseq [ingested lst]
(load-one! (enrich ingested) nil)))
(entity-id? first-eid) ; proper entity id, match by them
(do (t2/delete! model backward-fk parent-id :entity_id [:not-in (map :entity_id lst)])
(doseq [ingested lst
:let [ingested (enrich ingested)
local (lookup-by-id model (entity-id model-name ingested))]]
(load-one! ingested local)))
:else ; identity hash
(let [incoming (set (map #(entity-id model-name %) lst))
local (->> (t2/reducible-select model backward-fk parent-id)
(into [] (map t2.realize/realize))
(m/index-by identity-hash))
to-delete (into [] (comp (filter #(contains? incoming (key %)))
(map #(:id (val %))))
local)]
(t2/delete! model :id [:in (map :id to-delete)])
(doseq [ingested lst]
(load-one! (enrich ingested) (get local (entity-id model-name ingested))))))))}))
(defn parent-ref "Transformer for parent id for nested entities" []
{::fk true :export (constantly nil) :import identity})
(defn date "Transformer to parse the dates" []
{:export identity :import #(if (string? %) (u.date/parse %) %)})
;;; ## Memoizing appdb lookups
(defmacro with-cache
......
src/metabase/util.cljc
+ 5
− 0
View file @ d5b25cd9
......@@ -1017,3 +1017,8 @@
(let [buf (js/Uint8Array. max-length-bytes)
result (.encodeInto (js/TextEncoder.) s buf)] ;; JS obj {read: chars_converted, write: bytes_written}
(subs s 0 (.-read result)))))
(defn rfirst
"Return first item from Reducible"
[reducible]
(reduce (fn [_ fst] (reduced fst)) nil reducible))
test/metabase/test/generate.clj
+ 2
− 1
View file @ d5b25cd9
......@@ -170,7 +170,8 @@
(s/def ::pulse-card (s/keys :req-un [::id ::position]))
(s/def ::channel_type ::not-empty-string)
(s/def ::schedule_type ::not-empty-string)
(s/def ::schedule_type (s/with-gen (s/and string? #(contains? #{"hourly" "weekly" "monthly"} %))
#(gen/elements ["hourly" "weekly" "monthly"])))
(s/def ::pulse-channel (s/keys :req-un [::id ::channel_type ::details ::schedule_type]))
(s/def ::pulse-channel-recipient (s/keys :req-un [::id]))
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment