Skip to content
Snippets Groups Projects
Unverified Commit 118b5c35 authored by Braden Shepherdson's avatar Braden Shepherdson Committed by GitHub
Browse files

Serialization for Dashboard, Dashboard Cards (#23804)

parent 5df0e1d1
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Showing
with 192 additions and 25 deletions
enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj
+ 2
− 0
View file @ 118b5c35
......@@ -4,6 +4,8 @@
"The list of models which are exported by serialization. Used for production code and by tests."
["Card"
"Collection"
"Dashboard"
"DashboardCard"
"Database"
"Field"
"Setting"
......
enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj
+ 77
− 23
View file @ 118b5c35
......@@ -2,7 +2,7 @@
(:require [clojure.test :refer :all]
[metabase-enterprise.serialization.test-util :as ts]
[metabase-enterprise.serialization.v2.extract :as extract]
[metabase.models :refer [Card Collection Database Table User]]
[metabase.models :refer [Card Collection Dashboard DashboardCard Database Table User]]
[metabase.models.serialization.base :as serdes.base]))
(defn- select-one [model-name where]
......@@ -75,29 +75,51 @@
(deftest dashboard-and-cards-test
(ts/with-empty-h2-app-db
(ts/with-temp-dpc [Collection [{coll-id :id
coll-eid :entity_id
coll-slug :slug} {:name "Some Collection"}]
User [{mark-id :id} {:first_name "Mark"
:last_name "Knopfler"
:email "mark@direstrai.ts"}]
Database [{db-id :id} {:name "My Database"}]
Table [{no-schema-id :id} {:name "Schemaless Table" :db_id db-id}]
Table [{schema-id :id} {:name "Schema'd Table"
:db_id db-id
:schema "PUBLIC"}]
coll-eid :entity_id} {:name "Some Collection"}]
User [{mark-id :id} {:first_name "Mark"
:last_name "Knopfler"
:email "mark@direstrai.ts"}]
User [{dave-id :id} {:first_name "David"
:last_name "Knopfler"
:email "david@direstrai.ts"}]
Collection [{mark-coll-eid :entity_id} {:name "MK Personal"
:personal_owner_id mark-id}]
Collection [{dave-coll-id :id
dave-coll-eid :entity_id} {:name "DK Personal"
:personal_owner_id dave-id}]
Database [{db-id :id} {:name "My Database"}]
Table [{no-schema-id :id} {:name "Schemaless Table" :db_id db-id}]
Table [{schema-id :id} {:name "Schema'd Table"
:db_id db-id
:schema "PUBLIC"}]
Card [{c1-id :id
c1-eid :entity_id} {:name "Some Question"
:database_id db-id
:table_id no-schema-id
:collection_id coll-id
:creator_id mark-id
:dataset_query "{\"json\": \"string values\"}"}]
c1-eid :entity_id} {:name "Some Question"
:database_id db-id
:table_id no-schema-id
:collection_id coll-id
:creator_id mark-id
:dataset_query "{\"json\": \"string values\"}"}]
Card [{c2-id :id
c2-eid :entity_id} {:name "Second Question"
:database_id db-id
:table_id schema-id
:collection_id coll-id
:creator_id mark-id}]]
c2-eid :entity_id} {:name "Second Question"
:database_id db-id
:table_id schema-id
:collection_id coll-id
:creator_id mark-id}]
Dashboard [{dash-id :id
dash-eid :entity_id} {:name "Shared Dashboard"
:collection_id coll-id
:creator_id mark-id
:parameters []}]
Dashboard [{other-dash-id :id
other-dash :entity_id} {:name "Dave's Dash"
:collection_id dave-coll-id
:creator_id mark-id
:parameters []}]
DashboardCard [{dc1-eid :entity_id} {:card_id c1-id
:dashboard_id dash-id}]
DashboardCard [{dc2-eid :entity_id} {:card_id c2-id
:dashboard_id other-dash-id}]]
(testing "table and database are extracted as [db schema table] triples"
(let [ser (serdes.base/extract-one "Card" {} (select-one "Card" [:= :id c1-id]))]
(is (= {:serdes/meta [{:model "Card" :id c1-eid}]
......@@ -128,4 +150,36 @@
{:model "Schema" :id "PUBLIC"}
{:model "Table" :id "Schema'd Table"}]
[{:model "Collection" :id coll-eid}]}
(set (serdes.base/serdes-dependencies ser))))))))))
(set (serdes.base/serdes-dependencies ser)))))))
(testing "collection filtering based on :user option"
(testing "only unowned collections are returned with no user"
(is (= ["Some Collection"]
(->> (serdes.base/extract-all "Collection" {})
(into [])
(map :name)))))
(testing "unowned collections and the personal one with a user"
(is (= #{coll-eid mark-coll-eid}
(by-model "Collection" (serdes.base/extract-all "Collection" {:user mark-id}))))
(is (= #{coll-eid dave-coll-eid}
(by-model "Collection" (serdes.base/extract-all "Collection" {:user dave-id}))))))
(testing "dashboards are filtered based on :user"
(testing "dashboards in unowned collections are always returned"
(is (= #{dash-eid}
(by-model "Dashboard" (serdes.base/extract-all "Dashboard" {}))))
(is (= #{dash-eid}
(by-model "Dashboard" (serdes.base/extract-all "Dashboard" {:user mark-id})))))
(testing "dashboards in personal collections are returned for the :user"
(is (= #{dash-eid other-dash}
(by-model "Dashboard" (serdes.base/extract-all "Dashboard" {:user dave-id}))))))
(testing "dashboard cards are filtered based on :user"
(testing "dashboard cards whose dashboards are in unowned collections are always returned"
(is (= #{dc1-eid}
(by-model "DashboardCard" (serdes.base/extract-all "DashboardCard" {}))))
(is (= #{dc1-eid}
(by-model "DashboardCard" (serdes.base/extract-all "DashboardCard" {:user mark-id})))))
(testing "dashboard cards whose dashboards are in personal collections are returned for the :user"
(is (= #{dc1-eid dc2-eid}
(by-model "DashboardCard" (serdes.base/extract-all "DashboardCard" {:user dave-id})))))))))
enterprise/backend/test/metabase_enterprise/serialization/v2/yaml_test.clj
+ 33
− 1
View file @ 118b5c35
......@@ -115,7 +115,11 @@
{:database_id (keyword (str "db" db))
:table_id (keyword (str "t" (+ t (* 10 db))))
:collection_id (random-key "coll" 100)
:creator_id (random-key "u" 10)})}]]})
:creator_id (random-key "u" 10)})}]]
:dashboard [[100 {:refs {:collection_id (random-key "coll" 100)
:creator_id (random-key "u" 10)}}]]
:dashboard-card [[300 {:refs {:card_id (random-key "c" 100)
:dashboard_id (random-key "d" 100)}}]]})
(let [extraction (into [] (extract/extract-metabase {}))
entities (reduce (fn [m entity]
(update m (-> entity :serdes/meta last :model)
......@@ -187,6 +191,34 @@
(update :updated_at u.date/format))
(yaml/from-file (io/file dump-dir "Card" filename))))))
(testing "for dashboards"
(is (= 100 (count (dir->file-set (io/file dump-dir "Dashboard")))))
(doseq [{:keys [collection_id creator_id entity_id]
:as dash} (get entities "Dashboard")
:let [filename (str entity_id ".yaml")]]
(is (= (-> dash
(dissoc :serdes/meta)
(update :created_at u.date/format)
(update :updated_at u.date/format))
(yaml/from-file (io/file dump-dir "Dashboard" filename))))))
(testing "for dashboard cards"
(is (= 300
(reduce + (for [dash (get entities "Dashboard")
:let [card-dir (io/file dump-dir "Dashboard" (:entity_id dash) "DashboardCard")]]
(if (.exists card-dir)
(count (dir->file-set card-dir))
0)))))
(doseq [{:keys [dashboard_id entity_id]
:as dashcard} (get entities "DashboardCard")
:let [filename (str entity_id ".yaml")]]
(is (= (-> dashcard
(dissoc :serdes/meta)
(update :created_at u.date/format)
(update :updated_at u.date/format))
(yaml/from-file (io/file dump-dir "Dashboard" dashboard_id "DashboardCard" filename))))))
(testing "for settings"
(is (= (into {} (for [{:keys [key value]} (get entities "Setting")]
[key value]))
......
src/metabase/models/dashboard.clj
+ 39
− 0
View file @ 118b5c35
......@@ -17,6 +17,7 @@
[metabase.models.pulse-card :as pulse-card :refer [PulseCard]]
[metabase.models.revision :as revision]
[metabase.models.revision.diff :refer [build-sentence]]
[metabase.models.serialization.base :as serdes.base]
[metabase.models.serialization.hash :as serdes.hash]
[metabase.moderation :as moderation]
[metabase.public-settings :as public-settings]
......@@ -414,3 +415,41 @@
{(:parameter_id param) #{(assoc param :dashcard dashcard)}}))]
(into {} (for [{param-key :id, :as param} (:parameters dashboard)]
[(u/qualified-name param-key) (assoc param :mappings (get param-key->mappings param-key))]))))
;;; +----------------------------------------------------------------------------------------------------------------+
;;; | SERIALIZATION |
;;; +----------------------------------------------------------------------------------------------------------------+
(defmethod serdes.base/extract-query "Dashboard" [_ {:keys [user]}]
;; TODO This join over the subset of collections this user can see is shared by a few things - factor it out?
(serdes.base/raw-reducible-query
"Dashboard"
{:select [:dash.*]
:from [[:report_dashboard :dash]]
:left-join [[:collection :coll] [:= :coll.id :dash.collection_id]]
:where (if user
[:or [:= :coll.personal_owner_id user] [:is :coll.personal_owner_id nil]]
[:is :coll.personal_owner_id nil])}))
;; TODO Maybe nest collections -> dashboards -> dashcards?
(defmethod serdes.base/extract-one "Dashboard"
[_ _ {:keys [collection_id creator_id] :as dash}]
(let [email (db/select-one-field :email 'User :id creator_id)
coll (db/select-one 'Collection :id collection_id)
{collection-eid :id} (serdes.base/infer-self-path "Collection" coll)]
(-> (serdes.base/extract-one-basics "Dashboard" dash)
(assoc :collection_id collection-eid
:creator_id email))))
(defmethod serdes.base/load-xform "Dashboard"
[{email :creator_id
collection-eid :collection_id
:as dash}]
(let [coll-id (serdes.base/lookup-by-id 'Collection collection-eid)
user-id (db/select-one-id 'User :email email)]
(-> dash
(assoc :collection_id coll-id
:creator_id user-id))))
(defmethod serdes.base/serdes-dependencies "Dashboard"
[{:keys [collection_id]}]
[[{:model "Collection" :id collection_id}]])
src/metabase/models/dashboard_card.clj
+ 36
− 0
View file @ 118b5c35
......@@ -6,6 +6,7 @@
[metabase.models.dashboard-card-series :refer [DashboardCardSeries]]
[metabase.models.interface :as mi]
[metabase.models.pulse-card :refer [PulseCard]]
[metabase.models.serialization.base :as serdes.base]
[metabase.models.serialization.hash :as serdes.hash]
[metabase.util :as u]
[metabase.util.schema :as su]
......@@ -203,3 +204,38 @@
(db/delete! PulseCard :dashboard_card_id (:id dashboard-card))
(db/delete! DashboardCard :id (:id dashboard-card)))
(events/publish-event! :dashboard-remove-cards {:id id :actor_id user-id :dashcards [dashboard-card]})))
;;; ----------------------------------------------- SERIALIZATION ----------------------------------------------------
(defmethod serdes.base/extract-query "DashboardCard" [_ {:keys [user]}]
;; TODO This join over the subset of collections this user can see is shared by a few things - factor it out?
(serdes.base/raw-reducible-query
"DashboardCard"
{:select [:dc.*]
:from [[:report_dashboardcard :dc]]
:left-join [[:report_dashboard :dash] [:= :dash.id :dc.dashboard_id]
[:collection :coll] [:= :coll.id :dash.collection_id]]
:where (if user
[:or [:= :coll.personal_owner_id user] [:is :coll.personal_owner_id nil]]
[:is :coll.personal_owner_id nil])}))
(defmethod serdes.base/serdes-dependencies "DashboardCard" [{:keys [card_id dashboard_id]}]
[[{:model "Dashboard" :id dashboard_id}]
[{:model "Card" :id card_id}]])
(defmethod serdes.base/serdes-generate-path "DashboardCard" [_ dashcard]
[(serdes.base/infer-self-path "Dashboard" (db/select-one 'Dashboard :id (:dashboard_id dashcard)))
(serdes.base/infer-self-path "DashboardCard" dashcard)])
(defmethod serdes.base/extract-one "DashboardCard"
[_ _ {:keys [card_id dashboard_id] :as dashcard}]
(let [card (db/select-one 'Card :id card_id)
dash (db/select-one 'Dashboard :id dashboard_id)]
(-> (serdes.base/extract-one-basics "DashboardCard" dashcard)
(assoc :card_id (or (:entity_id card) (serdes.hash/identity-hash card)))
(assoc :dashboard_id (or (:entity_id dash) (serdes.hash/identity-hash dash))))))
(defmethod serdes.base/load-xform "DashboardCard"
[{:keys [card_id dashboard_id] :as dashcard}]
(-> (serdes.base/load-xform-basics dashcard)
(assoc :card_id (serdes.base/lookup-by-id 'Card card_id))
(assoc :dashboard_id (serdes.base/lookup-by-id 'Dashboard dashboard_id))))
test/metabase/test/generate.clj
+ 5
− 1
View file @ 118b5c35
......@@ -116,7 +116,11 @@
;; * native-query-snippet
(s/def ::content ::not-empty-string)
(s/def ::parameters #{[{:id "a"}]})
(s/def :parameter/id ::not-empty-string)
(s/def :parameter/type ::base_type)
(s/def ::parameter (s/keys :req-un [:parameter/id :parameter/type]))
(s/def ::parameters (s/coll-of ::parameter))
;; * pulse
(s/def ::row pos-int?)
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment