Code owners
Assign users and groups as approvers for specific file changes. Learn more.
e2e_test.clj 48.88 KiB
(ns ^:mb/once metabase-enterprise.serialization.v2.e2e-test
(:require
[clojure.java.io :as io]
[clojure.test :refer :all]
[medley.core :as m]
[metabase-enterprise.serialization.cmd :as cmd]
[metabase-enterprise.serialization.test-util :as ts]
[metabase-enterprise.serialization.v2.extract :as extract]
[metabase-enterprise.serialization.v2.ingest :as ingest]
[metabase-enterprise.serialization.v2.load :as serdes.load]
[metabase-enterprise.serialization.v2.storage :as storage]
[metabase.models :refer [Card
Collection
Dashboard
DashboardCard
Database
Field
ParameterCard
Table]]
[metabase.models.action :as action]
[metabase.models.serialization :as serdes]
[metabase.models.setting :as setting]
[metabase.test :as mt]
[metabase.test.generate :as test-gen]
[metabase.util :as u]
[metabase.util.yaml :as yaml]
[reifyhealth.specmonstah.core :as rs]
[toucan2.core :as t2]
[toucan2.tools.with-temp :as t2.with-temp])
(:import
(java.io File)
(java.nio.file Path)))
(set! *warn-on-reflection* true)
(defn- dir->contents-set [p ^File dir]
(->> dir
.listFiles
(filter p)
(map #(.getName ^File %))
set))
(defn- dir->file-set [^File dir]
(dir->contents-set #(.isFile ^File %) dir))
(defn- dir->dir-set [^File dir]
(dir->contents-set #(.isDirectory ^File %) dir))
(defn- subdirs [^File dir]
(->> dir
.listFiles
(remove #(.isFile ^File %))))
(defn- by-model [entities model-name]
(filter #(-> % :serdes/meta last :model (= model-name))
entities))
(defn- by-eid [entities entity-id]
(u/seek #(= (:entity_id %) entity-id) entities))
(defn- collections [dir]
(for [coll-dir (subdirs dir)
:when (->> ["cards" "dashboards" "timelines"]
(map #(io/file coll-dir %))
(filter #(= % coll-dir))
empty?)]
coll-dir))
(defn- file-set [^File dir]
(let [^Path base (.toPath dir)] (set (for [^File file (file-seq dir)
:when (.isFile file)
:let [rel (.relativize base (.toPath file))]]
(mapv str rel)))))
(defn- random-keyword
([prefix n] (random-keyword prefix n 0))
([prefix n floor] (keyword (str (name prefix) (+ floor (rand-int n))))))
(defn- random-fks
"Generates a specmonstah query with the :refs populated with the randomized bindings.
`(random-fks {:spec-gen {:foo :bar}}
{:creator_id [:u 10]
:db_id [:db 20 15]})`
this will return a query like:
`{:spec-gen {:foo :bar}
:refs {:creator_id :u6 :db_id 17}}`
The bindings map has the same keys as `:refs`, but the values are `[base-keyword width]` pairs or
`[base-keyword width floor]` triples. These are passed to [[random-keyword]]."
[base bindings]
(update base :refs merge (m/map-vals #(apply random-keyword %) bindings)))
(defn- many-random-fks [n base bindings]
(vec (repeatedly n #(vector 1 (random-fks base bindings)))))
(defn- table->db [{:keys [table_id] :as refs}]
(let [table-number (-> table_id
name
(subs 1)
(Integer/parseInt))]
(assoc refs :database_id (keyword (str "db" (quot table-number 10))))))
(defn- clean-entity
"Removes any comparison-confounding fields, like `:created_at`."
[entity]
(dissoc entity :created_at :result_metadata :metadata_sync_schedule :cache_field_values_schedule))
#_{:clj-kondo/ignore [:metabase/i-like-making-cams-eyes-bleed-with-horrifically-long-tests]}
(deftest e2e-storage-ingestion-test
(ts/with-random-dump-dir [dump-dir "serdesv2-"]
(let [extraction (atom nil)
entities (atom nil)]
(ts/with-dbs [source-db dest-db]
;; TODO Generating some nested collections would make these tests more robust, but that's difficult.
;; There are handwritten tests for storage and ingestion that check out the nesting, at least.
(ts/with-db source-db
(testing "insert"
(test-gen/insert!
{;; Actions are special case where there is a 1:1 relationship between an action and an action subtype (query, implicit, or http)
;; We generate 10 actions for each subtype, and 10 of each subtype.
;; actions 0-9 are query actions, 10-19 are implicit actions, and 20-29 are http actions.
:action (apply concat
(for [type [:query :implicit :http]]
(many-random-fks 10
{:spec-gen {:type type}}
{:model_id [:sm 10]
:creator_id [:u 10]})))
:query-action (map-indexed
(fn [idx x]
(assoc-in x [1 :refs :action_id] (keyword (str "action" idx))))
(many-random-fks 10 {} {:database_id [:db 10]}))
:implicit-action (map-indexed
(fn [idx x]
(update-in x [1 :refs]
(fn [refs]
(assoc refs :action_id (keyword (str "action" (+ 10 idx)))))))
(many-random-fks 10 {} {}))
:http-action (map-indexed
(fn [idx x] (update-in x [1 :refs]
(fn [refs]
(assoc refs :action_id (keyword (str "action" (+ 20 idx)))))))
(many-random-fks 10 {} {}))
:collection [[100 {:refs {:personal_owner_id ::rs/omit}}]
[10 {:refs {:personal_owner_id ::rs/omit}
:spec-gen {:namespace :snippets}}]]
:database [[10]]
;; Tables are special - we define table 0-9 under db0, 10-19 under db1, etc. The :card spec below
;; depends on this relationship.
:table (into [] (for [db [:db0 :db1 :db2 :db3 :db4 :db5 :db6 :db7 :db8 :db9]]
[10 {:refs {:db_id db}}]))
:field (many-random-fks 1000 {} {:table_id [:t 100]})
:core-user [[100]]
:card (mapv #(update-in % [1 :refs] table->db)
(many-random-fks
100
{:spec-gen {:dataset_query {:database 1
:type :native
:native {:query "SELECT * FROM whatever;"}}
:type :model}}
{:table_id [:t 100]
:collection_id [:coll 100]
:creator_id [:u 10]}))
;; Simple model is primary used for actions.
;; We can't use :card for actions because implicit actions require the model's query to contain
;; nothing but a source table
:simple-model (mapv #(update-in % [1 :refs] table->db)
(many-random-fks
10
{:spec-gen {:dataset_query {:database 1
:query {:source-table 3}
:type :query}
:type :model}}
{:table_id [:t 10]
:collection_id [:coll 10]
:creator_id [:u 10]}))
:dashboard (concat (many-random-fks 100 {} {:collection_id [:coll 100]
:creator_id [:u 10]})
;; create some root collection dashboards
(many-random-fks 50 {} {:creator_id [:u 10]}))
:dashboard-card (many-random-fks 300 {} {:card_id [:c 100]
:dashboard_id [:d 100]})
:dimension (vec (concat
;; 20 with both IDs set
(many-random-fks 20 {}
{:field_id [:field 1000]
:human_readable_field_id [:field 1000]})
;; 20 with just :field_id
(many-random-fks 20 {:refs {:human_readable_field_id ::rs/omit}}
{:field_id [:field 1000]})))
:metric (many-random-fks 30 {:spec-gen {:definition {:aggregation [[:count]]
:source-table 9}}}
{:table_id [:t 100]
:creator_id [:u 10]})
:segment (many-random-fks 30 {:spec-gen {:definition {:filter [:!= [:field 60 nil] 50],
:source-table 4}}}
{:table_id [:t 100]
:creator_id [:u 10]})
:native-query-snippet (many-random-fks 10 {} {:creator_id [:u 10]
:collection_id [:coll 10 100]})
:timeline (many-random-fks 10 {} {:creator_id [:u 10]
:collection_id [:coll 100]})
:timeline-event (many-random-fks 90 {} {:timeline_id [:timeline 10]})}))
(is (= 101 (count (t2/select-fn-set :email 'User)))) ; +1 for the internal user
(testing "extraction"
(reset! extraction (serdes/with-cache (into [] (extract/extract {}))))
(reset! entities (reduce (fn [m entity] (update m (-> entity :serdes/meta last :model)
(fnil conj []) entity))
{} @extraction))
(is (= 110 (-> @entities (get "Collection") count))))
(testing "storage"
(storage/store! (seq @extraction) dump-dir)
(testing "for Actions"
(is (= 30 (count (dir->file-set (io/file dump-dir "actions"))))))
(testing "for Collections"
(is (= 110 (count (for [f (file-set (io/file dump-dir))
:when (and (= (first f) "collections")
(let [[a b] (take-last 2 f)]
(= b (str a ".yaml"))))]
f)))
"which all go in collections/, even the snippets ones"))
(testing "for Databases"
(is (= 10 (count (dir->dir-set (io/file dump-dir "databases"))))))
(testing "for Tables"
(is (= 100
(reduce + (for [db (get @entities "Database")
:let [tables (dir->dir-set (io/file dump-dir "databases" (:name db) "tables"))]]
(count tables))))
"Tables are scattered, so the directories are harder to count"))
(testing "for Fields"
(is (= 1000
(reduce + (for [db (get @entities "Database")
table (subdirs (io/file dump-dir "databases" (:name db) "tables"))]
(->> (io/file table "fields")
dir->file-set
count))))
"Fields are scattered, so the directories are harder to count"))
(testing "for cards"
;; 100 from card, and 10 from simple-model
(is (= 110 (->> (io/file dump-dir "collections")
collections
(map (comp count dir->file-set #(io/file % "cards")))
(reduce +)))))
(testing "for dashboards"
(is (= 150 (->> (io/file dump-dir "collections")
collections
(map (comp count dir->file-set #(io/file % "dashboards")))
(reduce +)))))
(testing "for timelines"
(is (= 10 (->> (io/file dump-dir "collections")
collections
(map (comp count dir->file-set #(io/file % "timelines")))
(reduce +)))))
(testing "for metrics"
(is (= 30 (reduce + (for [db (dir->dir-set (io/file dump-dir "databases"))
table (dir->dir-set (io/file dump-dir "databases" db "tables"))
:let [metrics-dir (io/file dump-dir "databases" db "tables" table "metrics")]
:when (.exists metrics-dir)]
(count (dir->file-set metrics-dir)))))))
(testing "for segments"
(is (= 30 (reduce + (for [db (dir->dir-set (io/file dump-dir "databases"))
table (dir->dir-set (io/file dump-dir "databases" db "tables"))
:let [segments-dir (io/file dump-dir "databases" db "tables" table "segments")]
:when (.exists segments-dir)]
(count (dir->file-set segments-dir)))))))
(testing "for native query snippets"
(is (= 10 (->> (io/file dump-dir "snippets")
collections
(map (comp count dir->file-set))
(reduce +)))))
(testing "for settings"
(is (.exists (io/file dump-dir "settings.yaml")))))
(testing "ingest and load"
(ts/with-db dest-db
(testing "ingested set matches extracted set"
(let [extracted-set (set (map (comp #'ingest/strip-labels serdes/path) @extraction))]
(is (= (count extracted-set)
(count @extraction)))
(is (= extracted-set
(set (ingest/ingest-list (ingest/ingest-yaml dump-dir)))))))
(testing "doing ingestion"
(is (serdes/with-cache (serdes.load/load-metabase! (ingest/ingest-yaml dump-dir)))
"successful"))
(testing "for Actions"
(doseq [{:keys [entity_id] :as coll} (get @entities "Action")]
(is (= (clean-entity coll)
(->> (t2/select-one 'Action :entity_id entity_id)
(@#'action/hydrate-subtype)
(serdes/extract-one "Action" {})
clean-entity)))))
(testing "for Collections"
(doseq [{:keys [entity_id] :as coll} (get @entities "Collection")]
(is (= (clean-entity coll)
(->> (t2/select-one 'Collection :entity_id entity_id)
(serdes/extract-one "Collection" {})
clean-entity)))))
(testing "for Databases"
(doseq [{:keys [name] :as db} (get @entities "Database")]
(is (= (assoc (clean-entity db) :initial_sync_status "complete")
(->> (t2/select-one 'Database :name name)
(serdes/extract-one "Database" {})
clean-entity)))))
(testing "for Tables"
(doseq [{:keys [db_id name] :as coll} (get @entities "Table")]
(is (= (clean-entity coll)
(->> (t2/select-one-fn :id 'Database :name db_id)
(t2/select-one 'Table :name name :db_id)
(serdes/extract-one "Table" {})
clean-entity)))))
(testing "for Fields"
(doseq [{[db schema table] :table_id name :name :as coll} (get @entities "Field")]
(is (nil? schema))
(is (= (clean-entity coll)
(->> (t2/select-one-fn :id 'Database :name db)
(t2/select-one-fn :id 'Table :schema schema :name table :db_id)
(t2/select-one 'Field :name name :table_id)
(serdes/extract-one "Field" {})
clean-entity)))))
(testing "for cards"
(doseq [{:keys [entity_id] :as card} (get @entities "Card")]
(is (= (clean-entity card)
(->> (t2/select-one 'Card :entity_id entity_id)
(serdes/extract-one "Card" {})
clean-entity)))))
(testing "for dashboards"
(doseq [{:keys [entity_id] :as dash} (get @entities "Dashboard")]
(is (= (clean-entity dash)
(->> (t2/select-one 'Dashboard :entity_id entity_id)
(serdes/extract-one "Dashboard" {})
clean-entity)))))
(testing "for dashboard cards"
(doseq [{:keys [entity_id] :as dashcard} (get @entities "DashboardCard")]
(is (= (clean-entity dashcard)
(->> (t2/select-one 'DashboardCard :entity_id entity_id)
(serdes/extract-one "DashboardCard" {})
clean-entity)))))
(testing "for dimensions"
(doseq [{:keys [entity_id] :as dim} (get @entities "Dimension")]
(is (= (clean-entity dim)
(->> (t2/select-one 'Dimension :entity_id entity_id)
(serdes/extract-one "Dimension" {})
clean-entity)))))
(testing "for metrics"
(doseq [{:keys [entity_id] :as metric} (get @entities "Metric")]
(is (= (clean-entity metric)
(->> (t2/select-one 'Metric :entity_id entity_id)
(serdes/extract-one "Metric" {})
clean-entity)))))
(testing "for segments"
(doseq [{:keys [entity_id] :as segment} (get @entities "Segment")]
(is (= (clean-entity segment)
(->> (t2/select-one 'Segment :entity_id entity_id)
(serdes/extract-one "Segment" {})
clean-entity)))))
(testing "for native query snippets"
(doseq [{:keys [entity_id] :as snippet} (get @entities "NativeQuerySnippet")]
(is (= (clean-entity snippet)
(->> (t2/select-one 'NativeQuerySnippet :entity_id entity_id)
(serdes/extract-one "NativeQuerySnippet" {})
clean-entity)))))
(testing "for timelines and events"
(doseq [{:keys [entity_id] :as timeline} (get @entities "Timeline")]
(is (= (clean-entity timeline)
(->> (t2/select-one 'Timeline :entity_id entity_id)
(serdes/extract-one "Timeline" {})
clean-entity)))))
(testing "for settings"
(let [settings (get @entities "Setting")]
(is (every? @#'setting/export?
(set (map (comp symbol :key) settings))))
(is (= (into {} (for [{:keys [key value]} settings]
[key value]))
(yaml/from-file (io/file dump-dir "settings.yaml")))))))))))))
;; This is a seperate test instead of a `testing` block inside `e2e-storage-ingestion-test`
;; because it's quite tricky to set up the generative test to generate parameters with source is card
(deftest card-and-dashboard-has-parameter-with-source-is-card-test
(testing "Dashboard and Card that has parameter with source is a card must be deserialized correctly"
(ts/with-random-dump-dir [dump-dir "serdesv2-"]
(ts/with-dbs [source-db dest-db]
(ts/with-db source-db
;; preparation
(mt/test-helpers-set-global-values!
(mt/with-temp
[Database db1s {:name "my-db"}
Collection coll1s {:name "My Collection"}
Table table1s {:name "CUSTOMERS" :db_id (:id db1s)}
Field field1s {:name (mt/random-name)
:table_id (:id table1s)}
Card card1s {:name (mt/random-name)}
Card card2s {:name "Card with parameter"
:database_id (:id db1s)
:table_id (:id table1s)
:collection_id (:id coll1s)
:parameters [{:id "abc"
:type "category"
:name "CATEGORY"
:values_source_type "card"
;; card_id is in a different collection with dashboard's collection
:values_source_config {:card_id (:id card1s)
:value_field [:field (:id field1s) nil]}}]}
Dashboard dash1s {:name (mt/random-name)
:collection_id (:id coll1s)
:parameters [{:id "abc"
:type "category"
:name "CATEGORY"
:values_source_type "card"
;; card_id is in a different collection with dashboard's collection
:values_source_config {:card_id (:id card1s)
:value_field [:field (:id field1s) nil]}}]}]
(testing "make sure we insert ParameterCard when insert Dashboard/Card"
;; one for parameter on card card2s, and one for parmeter on dashboard dash1s
(is (= 2 (t2/count ParameterCard))))
(testing "extract and store"
(let [extraction (serdes/with-cache (into [] (extract/extract {})))]
(is (= [{:id "abc",
:name "CATEGORY",
:type :category,
:values_source_config {:card_id (:entity_id card1s),
:value_field [:field
["my-db" nil "CUSTOMERS" (:name field1s)]
nil]},
:values_source_type "card"}]
(:parameters (first (by-model extraction "Dashboard")))))
(is (= [{:id "abc",
:name "CATEGORY",
:type :category,
:values_source_config {:card_id (:entity_id card1s),
:value_field [:field
["my-db" nil "CUSTOMERS" (:name field1s)]
nil]},
:values_source_type "card"}]
(:parameters (by-eid extraction (:entity_id card2s)))))
(storage/store! (seq extraction) dump-dir)))
(testing "ingest and load"
(ts/with-db dest-db
;; ingest
(testing "doing ingestion"
(is (serdes/with-cache (serdes.load/load-metabase! (ingest/ingest-yaml dump-dir)))
"successful"))
(let [dash1d (t2/select-one Dashboard :name (:name dash1s))
card1d (t2/select-one Card :name (:name card1s))
card2d (t2/select-one Card :name (:name card2s))
field1d (t2/select-one Field :name (:name field1s))]
(testing "parameter on dashboard is loaded correctly"
(is (= {:card_id (:id card1d),
:value_field [:field (:id field1d) nil]}
(-> dash1d
:parameters
first :values_source_config)))
(is (some? (t2/select-one 'ParameterCard :parameterized_object_type "dashboard" :parameterized_object_id (:id dash1d)))))
(testing "parameter on card is loaded correctly"
(is (= {:card_id (:id card1d),
:value_field [:field (:id field1d) nil]}
(-> card2d
:parameters
first
:values_source_config)))
(is (some? (t2/select-one 'ParameterCard :parameterized_object_type "card" :parameterized_object_id (:id card2d)))))))))))))))
(deftest dashcards-with-link-cards-test
(ts/with-random-dump-dir [dump-dir "serdesv2-"]
(ts/with-dbs [source-db dest-db]
(ts/with-db source-db
(let [link-card-viz-setting (fn [model id]
{:virtual_card {:display "link"}
:link {:entity {:id id
:model model}}})
dashboard->link-cards (fn [dashboard]
(map #(get-in % [:visualization_settings :link :entity]) (:dashcards dashboard)))]
(t2.with-temp/with-temp
[Collection {coll-id :id
coll-name :name
coll-eid :entity_id} {:name "Link collection"
:description "Linked Collection"}
Database {db-id :id
db-name :name} {:name "Linked database"
:description "Linked database desc"}
Table {table-id :id
table-name :name} {:db_id db-id
:schema "Public"
:name "Linked table"
:description "Linked table desc"}
Card {card-id :id
card-name :name
card-eid :entity_id} {:name "Linked card"
:description "Linked card desc"
:display "bar"}
Card {model-id :id
model-name :name
model-eid :entity_id} {:type :model
:name "Linked model"
:description "Linked model desc"
:display "table"}
Dashboard {dash-id :id
dash-name :name
dash-eid :entity_id} {:name "Linked Dashboard"
:collection_id coll-id
:description "Linked Dashboard desc"}
Dashboard {dashboard-id :id
dashboard-name :name} {:name "Test Dashboard"
:collection_id coll-id}
DashboardCard _ {:dashboard_id dashboard-id
:visualization_settings (link-card-viz-setting "collection" coll-id)}
DashboardCard _ {:dashboard_id dashboard-id
:visualization_settings (link-card-viz-setting "database" db-id)}
DashboardCard _ {:dashboard_id dashboard-id
:visualization_settings (link-card-viz-setting "table" table-id)}
DashboardCard _ {:dashboard_id dashboard-id
:visualization_settings (link-card-viz-setting "dashboard" dash-id)}
DashboardCard _ {:dashboard_id dashboard-id
:visualization_settings (link-card-viz-setting "card" card-id)}
DashboardCard _ {:dashboard_id dashboard-id
:visualization_settings (link-card-viz-setting "dataset" model-id)}]
(testing "extract and store"
(let [extraction (serdes/with-cache (into [] (extract/extract {}))) extracted-dashboard (first (filter #(= (:name %) "Test Dashboard") (by-model extraction "Dashboard")))]
(is (= [{:model "collection" :id coll-eid}
{:model "database" :id "Linked database"}
{:model "table" :id ["Linked database" "Public" "Linked table"]}
{:model "dashboard" :id dash-eid}
{:model "card" :id card-eid}
{:model "dataset" :id model-eid}]
(dashboard->link-cards extracted-dashboard)))
(is (= #{[{:id dash-eid :model "Dashboard"}]
[{:id coll-eid :model "Collection"}]
[{:id model-eid :model "Card"}]
[{:id card-eid :model "Card"}]
[{:id "Linked database" :model "Database"}]
[{:model "Database" :id "Linked database"}
{:model "Schema" :id "Public"}
{:model "Table" :id "Linked table"}]}
(set (serdes/dependencies extracted-dashboard))))
(storage/store! (seq extraction) dump-dir)))
(testing "ingest and load"
;; ingest
(ts/with-db dest-db
(testing "doing ingestion"
(is (serdes/with-cache (serdes.load/load-metabase! (ingest/ingest-yaml dump-dir)))
"successful"))
(doseq [[name model]
[[db-name 'Database]
[table-name 'Table]
[card-name 'Card]
[model-name 'Card]
[dash-name 'Dashboard]]]
(testing (format "model %s from link cards are loaded properly" model)
(is (some? (t2/select model :name name)))))
(testing "linkcards are loaded with correct fk"
(let [new-db-id (t2/select-one-pk Database :name db-name)
new-table-id (t2/select-one-pk Table :name table-name)
new-card-id (t2/select-one-pk Card :name card-name)
new-model-id (t2/select-one-pk Card :name model-name)
new-dash-id (t2/select-one-pk Dashboard :name dash-name)
new-coll-id (t2/select-one-pk Collection :name coll-name)]
(is (= [{:id new-coll-id :model "collection"}
{:id new-db-id :model "database"}
{:id new-table-id :model "table"}
{:id new-dash-id :model "dashboard"}
{:id new-card-id :model "card"}
{:id new-model-id :model "dataset"}]
(-> (t2/select-one Dashboard :name dashboard-name)
(t2/hydrate :dashcards)
dashboard->link-cards)))))))))))))
(deftest dashcards-with-series-test
(ts/with-random-dump-dir [dump-dir "serdesv2-"]
(ts/with-dbs [source-db dest-db]
(ts/with-db source-db
(mt/with-temp
[:model/Collection {coll-id :id} {:name "Some Collection"}
:model/Card {c1-id :id :as c1} {:name "Some Question", :collection_id coll-id}
:model/Card {c2-id :id :as c2} {:name "Series Question A", :collection_id coll-id}
:model/Card {c3-id :id :as c3} {:name "Series Question B", :collection_id coll-id}
:model/Dashboard {dash-id :id :as dash} {:name "Shared Dashboard", :collection_id coll-id}
:model/DashboardCard {dc1-id :id} {:card_id c1-id, :dashboard_id dash-id}
:model/DashboardCard {dc2-id :id} {:card_id c2-id, :dashboard_id dash-id}
:model/DashboardCardSeries _ {:card_id c3-id, :dashboardcard_id dc1-id, :position 1}
:model/DashboardCardSeries _ {:card_id c2-id, :dashboardcard_id dc1-id, :position 0}]
(testing "sense check what hydrated dashcards look like on the source DB"
(let [hydrated-dashcards (-> (t2/select-one :model/Dashboard :name (:name dash)) (t2/hydrate [:dashcards :series])
:dashcards
(->> (m/index-by :id)))]
(is (=? {dc1-id {:series [{:id c2-id}
{:id c3-id}]}
dc2-id {:series []}}
hydrated-dashcards))))
(testing "extract and store"
(let [extraction (serdes/with-cache (into [] (extract/extract {})))]
(storage/store! (seq extraction) dump-dir)))
(testing "ingest and load"
(ts/with-db dest-db
(testing "doing ingestion"
(is (serdes/with-cache (serdes.load/load-metabase! (ingest/ingest-yaml dump-dir)))
"successful"))
(testing "Series are loaded correctly"
(let [new-c1-id (t2/select-one-pk :model/Card :name (:name c1))
new-c2-id (t2/select-one-pk :model/Card :name (:name c2))
new-c3-id (t2/select-one-pk :model/Card :name (:name c3))
new-dc1-id (t2/select-one-pk :model/DashboardCard :card_id new-c1-id)
new-dc2-id (t2/select-one-pk :model/DashboardCard :card_id new-c2-id)
new-hydrated-dashcards (-> (t2/select-one :model/Dashboard :name (:name dash))
(t2/hydrate [:dashcards :series])
:dashcards
(->> (m/index-by :id)))]
(is (some? new-c1-id))
(is (some? new-c2-id))
(is (some? new-c3-id))
(is (some? new-dc1-id))
(is (some? new-dc2-id))
(testing "Series hydrate on the dashboard correctly"
(is (=? {new-dc1-id {:series [{:id new-c2-id}
{:id new-c3-id}]}
new-dc2-id {:series []}}
new-hydrated-dashcards))))))))))))
(deftest dashboard-with-tabs-test
(testing "Dashboard with tabs must be deserialized correctly"
(ts/with-random-dump-dir [dump-dir "serdesv2-"]
(ts/with-dbs [source-db dest-db]
(ts/with-db source-db
;; preparation
(t2.with-temp/with-temp
[Dashboard {dashboard-id :id
dashboard-eid :entity_id} {:name "Dashboard with tab"}
Card {card-id-1 :id
card-eid-1 :entity_id} {:name "Card 1"}
Card {card-id-2 :id
card-eid-2 :entity_id} {:name "Card 2"}
:model/DashboardTab {tab-id-1 :id
tab-eid-1 :entity_id} {:name "Tab 1" :position 0 :dashboard_id dashboard-id}
:model/DashboardTab {tab-id-2 :id
tab-eid-2 :entity_id} {:name "Tab 2" :position 1 :dashboard_id dashboard-id}
DashboardCard _ {:dashboard_id dashboard-id
:card_id card-id-1
:dashboard_tab_id tab-id-1}
DashboardCard _ {:dashboard_id dashboard-id
:card_id card-id-2
:dashboard_tab_id tab-id-1}
DashboardCard _ {:dashboard_id dashboard-id
:card_id card-id-1
:dashboard_tab_id tab-id-2}
DashboardCard _ {:dashboard_id dashboard-id
:card_id card-id-2
:dashboard_tab_id tab-id-2}]
(let [extraction (serdes/with-cache (into [] (extract/extract {})))]
(storage/store! (seq extraction) dump-dir))
(testing "ingest and load"
(ts/with-db dest-db ;; ingest
(testing "doing ingestion"
(is (serdes/with-cache (serdes.load/load-metabase! (ingest/ingest-yaml dump-dir)))
"successful"))
(let [new-dashboard (-> (t2/select-one Dashboard :entity_id dashboard-eid)
(t2/hydrate :tabs :dashcards))
new-tab-id-1 (t2/select-one-pk :model/DashboardTab :entity_id tab-eid-1)
new-tab-id-2 (t2/select-one-pk :model/DashboardTab :entity_id tab-eid-2)
new-card-id-1 (t2/select-one-pk Card :entity_id card-eid-1)
new-card-id-2 (t2/select-one-pk Card :entity_id card-eid-2)]
(is (=? [{:id new-tab-id-1
:dashboard_id (:id new-dashboard)
:name "Tab 1"
:position 0}
{:id new-tab-id-2
:dashboard_id (:id new-dashboard)
:name "Tab 2"
:position 1}]
(:tabs new-dashboard)))
(is (=? [{:card_id new-card-id-1
:dashboard_id (:id new-dashboard)
:dashboard_tab_id new-tab-id-1}
{:card_id new-card-id-2
:dashboard_id (:id new-dashboard)
:dashboard_tab_id new-tab-id-1}
{:card_id new-card-id-1
:dashboard_id (:id new-dashboard)
:dashboard_tab_id new-tab-id-2}
{:card_id new-card-id-2
:dashboard_id (:id new-dashboard)
:dashboard_tab_id new-tab-id-2}]
(:dashcards new-dashboard))))))))))))
(deftest premium-features-test
(testing "with :serialization enabled on the token"
(ts/with-random-dump-dir [dump-dir "serdesv2-"]
(mt/with-premium-features #{:serialization}
(ts/with-dbs [source-db dest-db]
(ts/with-db source-db
;; preparation
(mt/with-temp [Dashboard _ {:name "some dashboard"}]
(testing "export (v2-dump) command"
(is (cmd/v2-dump! dump-dir {})
"works"))
(testing "import (v2-load) command"
(ts/with-db dest-db
(testing "doing ingestion"
(mt/with-temp [:model/User _ {}]
(is (cmd/v2-load! dump-dir {})
"works")))))))))))
(testing "without :serialization feature enabled"
(ts/with-random-dump-dir [dump-dir "serdesv2-"]
(mt/with-premium-features #{}
(ts/with-dbs [source-db dest-db]
(ts/with-db source-db
;; preparation
(t2.with-temp/with-temp [Dashboard _ {:name "some dashboard"}]
(testing "export (v2-dump) command"
(is (thrown-with-msg? Exception #"Please upgrade"
(cmd/v2-dump! dump-dir {}))
"throws"))
(testing "import (v2-load) command"
(ts/with-db dest-db
(testing "doing ingestion"
(mt/with-temp [:model/User _ {}]
(is (thrown-with-msg? Exception #"Please upgrade" (cmd/v2-load! dump-dir {}))
"throws"))))))))))))
(deftest pivot-export-test
(testing "Pivot table export and load correctly"
(let [old-ids (atom nil)
card1s (atom nil)]
(ts/with-random-dump-dir [dump-dir "serdesv2-"]
(ts/with-dbs [source-db dest-db]
(ts/with-db source-db
(mt/dataset test-data
;; ensuring field ids are stable by loading dataset in db first
(mt/db)
(mt/$ids nil
(t2.with-temp/with-temp
[Collection {coll-id :id} {:name "Pivot Collection"}
Card card {:name "Pivot Card"
:collection_id coll-id
:dataset_query {:type :query
:database (mt/id)
:query {:source-table $$orders
:aggregation [:sum [:field %orders.id nil]]
:breakout [[:field %orders.user_id nil]]}}
:visualization_settings
{:pivot_table.column_split
{:rows [[:field %people.name {:base-type :type/Text
:source-field %orders.user_id}]]
:columns [[:field %products.title {:base-type :type/Text
:source-field %orders.product_id}]]
:values [[:aggregation 0]]}
:column_settings
{(format "[\"ref\",[\"field\",%s,null]]" %people.name)
{:pivot_table.column_sort_order "descending"}}}}]
(reset! old-ids {:people.name %people.name
:orders.user_id %orders.user_id
:products.title %products.title
:orders.product_id %orders.product_id})
(reset! card1s card)
(storage/store! (extract/extract {}) dump-dir)))))
(ts/with-db dest-db
;; ensure there is something in db so that test-data gets different field ids for sure
(mt/dataset office-checkins
(mt/db))
(mt/dataset test-data
;; ensuring field ids are stable by loading dataset in db first
(mt/db)
(mt/$ids nil
(testing "Column ids are different in different dbs")
(is (not= @old-ids
{:people.name %people.name
:orders.user_id %orders.user_id
:products.title %products.title
:orders.product_id %orders.product_id}))
(serdes.load/load-metabase! (ingest/ingest-yaml dump-dir))
(let [viz (t2/select-one-fn :visualization_settings Card :entity_id (:entity_id @card1s))]
(testing "column ids inside pivot table transferred"
(is (= [[:field %people.name {:base-type :type/Text
:source-field %orders.user_id}]]
(get-in viz [:pivot_table.column_split :rows])))
(is (= [[:field %products.title {:base-type :type/Text
:source-field %orders.product_id}]]
(get-in viz [:pivot_table.column_split :columns]))))
(testing "column sort order restored"
(is (= "descending"
(get-in viz [:column_settings
(format "[\"ref\",[\"field\",%s,null]]" %people.name) :pivot_table.column_sort_order])))))))))))))
(deftest extra-files-test
(testing "Adding some extra files does not break deserialization"
(ts/with-random-dump-dir [dump-dir "serdesv2-"]
(mt/with-empty-h2-app-db
(let [coll (ts/create! Collection :name "coll")
_ (ts/create! Card :name "card" :collection_id (:id coll))]
(storage/store! (extract/extract {:no-settings true
:no-data-model true}) dump-dir)
(spit (io/file dump-dir "collections" ".hidden.yaml") "serdes/meta: [{do-not: read}]")
(spit (io/file dump-dir "collections" "unreadable.yaml") "\0")
(testing "No exceptions when loading despite unreadable files"
(let [logs (mt/with-log-messages-for-level ['metabase-enterprise :error]
(let [files (->> (#'ingest/ingest-all (io/file dump-dir))
(map (comp second second))
(map #(.getName ^File %))
set)]
(testing "Hidden YAML wasn't read even though it's not throwing errors"
(is (not (contains? files ".hidden.yaml"))))))]
(testing ".yaml files not containing valid yaml are just logged and do not break ingestion process"
(is (=? [[:error Throwable "Error reading file unreadable.yaml"]]
logs))))))))))