diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/backfill_ids.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/backfill_ids.clj
index 60e59658e5ed8e2d6ed163adbb7dd682ee5307b4..0d398c11e63ac0b019a2ec421f8b3eb3b29bbff5 100644
--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/backfill_ids.clj
+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/backfill_ids.clj
@@ -34,7 +34,7 @@
`entity_id` set. If the `entity_id` is NULL, it is set based on the [[serdes.hash/identity-hash]] for that
row."
[]
- (doseq [model-name serdes.models/exported-models
+ (doseq [model-name (concat serdes.models/exported-models serdes.models/inlined-models)
:let [model (db/resolve-model (symbol model-name))]
:when (has-entity-id? model)]
(backfill-ids-for model)))
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/extract.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/extract.clj
index 16735babf55d2ea03f8d64e0c4701914d98ad92d..ceab8d48c1c7f610bd5acde2b0b822a5d8eb4c1b 100644
--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/extract.clj
+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/extract.clj
@@ -30,7 +30,8 @@
set)]
(->> (concat unowned owned)
(map collection/descendant-ids)
- (reduce set/union top-ids))))
+ (reduce set/union top-ids)
+ (set/union #{nil}))))
(defn extract-metabase
"Extracts the appdb into a reducible stream of serializable maps, with `:serdes/meta` keys.
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/ingest/yaml.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/ingest/yaml.clj
index 39d42add793767c88a0d328e13decde7bc47b4b1..4260eefef1f517e7314b6486153f4b1b3053d06e 100644
--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/ingest/yaml.clj
+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/ingest/yaml.clj
@@ -4,7 +4,6 @@
(:require [clojure.java.io :as io]
[medley.core :as m]
[metabase-enterprise.serialization.v2.ingest :as ingest]
- [metabase-enterprise.serialization.v2.models :as models]
[metabase-enterprise.serialization.v2.utils.yaml :as u.yaml]
[metabase.util.date-2 :as u.date]
[yaml.core :as yaml]
@@ -22,7 +21,6 @@
; We return a path of 1 item, the setting itself.
[{:model "Setting" :id (name k)}])))
-
(defn- build-metas [^File root-dir ^File file]
(let [path-parts (u.yaml/path-split root-dir file)]
(if (= ["settings.yaml"] path-parts)
@@ -46,6 +44,9 @@
(sequential? obj) (mapv keywords obj)
:else obj))
+(defn- strip-labels [hierarchy]
+ (mapv #(dissoc % :label) hierarchy))
+
(defn- ingest-entity
"Given a hierarchy, read in the YAML file it identifies. Clean it up (eg. parsing timestamps) and attach the
hierarchy as `:serdes/meta`.
@@ -54,42 +55,53 @@
The labels are removed from the hierarchy attached at `:serdes/meta`, since the storage system might have damaged the
original labels by eg. truncating them to keep the file names from getting too long. The labels aren't used at all on
the loading side, so it's fine to drop them."
- [root-dir hierarchy]
- (let [unlabeled (mapv #(dissoc % :label) hierarchy)
- file (u.yaml/hierarchy->file root-dir hierarchy)] ; Use the original hierarchy for the filesystem.
- (-> (when (.exists file) file) ; If the returned file doesn't actually exist, replace it with nil.
+ [hierarchy ^File file]
+ (-> (when (.exists file) file) ; If the returned file doesn't actually exist, replace it with nil.
+
+ ;; No automatic keywords; it's too generous with what counts as a keyword and has a bug.
+ ;; See https://github.com/clj-commons/clj-yaml/issues/64
+ (yaml/from-file false)
+ keywords
+ read-timestamps
+ (assoc :serdes/meta (strip-labels hierarchy)))) ; But return the hierarchy without labels.
+
+(def ^:private legal-top-level-paths
+ "These are all the legal first segments of paths. This is used by ingestion to avoid `.git`, `.github`, `README.md`
+ and other such extras."
+ #{"collections" "databases" "snippets" "settings.yaml"})
- ;; No automatic keywords; it's too generous with what counts as a keyword and has a bug.
- ;; See https://github.com/clj-commons/clj-yaml/issues/64
- (yaml/from-file false)
- keywords
- read-timestamps
- (assoc :serdes/meta unlabeled)))) ; But return the hierarchy without labels.
+(defn- ingest-all [^File root-dir]
+ ;; This returns a map {unlabeled-hierarchy [original-hierarchy File]}.
+ (into {} (for [^File file (file-seq root-dir)
+ :when (and (.isFile file)
+ (let [rel (.relativize (.toPath root-dir) (.toPath file))]
+ (-> rel (.subpath 0 1) (.toString) legal-top-level-paths)))
+ hierarchy (build-metas root-dir file)]
+ [(strip-labels hierarchy) [hierarchy file]])))
-(deftype YamlIngestion [^File root-dir settings]
+(deftype YamlIngestion [^File root-dir settings cache]
ingest/Ingestable
(ingest-list [_]
- (let [model-set (set models/exported-models)]
- (eduction (comp (filter (fn [^File f] (.isFile f)))
- ;; The immediate parent directory should be a recognized model name.
- ;; If it's not, this may be in .git, or .github/actions/... or similar extra files.
- (filter (fn [^File f] (or (= (.getName f) "settings.yaml")
- (-> f
- (.getParentFile)
- (.getName)
- model-set))))
- (mapcat (partial build-metas root-dir)))
- (file-seq root-dir))))
+ (->> (or @cache
+ (reset! cache (ingest-all root-dir)))
+ vals
+ (map first)))
(ingest-one [_ abs-path]
+ (when-not @cache
+ (reset! cache (ingest-all root-dir)))
(let [{:keys [model id]} (first abs-path)]
(if (and (= (count abs-path) 1)
(= model "Setting"))
{:serdes/meta abs-path :key (keyword id) :value (get settings (keyword id))}
- (ingest-entity root-dir abs-path)))))
+ (->> abs-path
+ strip-labels
+ (get @cache)
+ second
+ (ingest-entity abs-path))))))
(defn ingest-yaml
"Creates a new Ingestable on a directory of YAML files, as created by
[[metabase-enterprise.serialization.v2.storage.yaml]]."
[root-dir]
- (->YamlIngestion (io/file root-dir) (yaml/from-file (io/file root-dir "settings.yaml"))))
+ (->YamlIngestion (io/file root-dir) (yaml/from-file (io/file root-dir "settings.yaml")) (atom nil)))
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj
index f4d7c94b71107668152a39ad99117ca24f7b6271..c5caad31a0288bf0cee0ff53fb3175d1e941a0ed 100644
--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj
+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj
@@ -10,10 +10,15 @@
"FieldValues"
"Metric"
"NativeQuerySnippet"
- "Pulse"
- "PulseCard"
- "PulseChannel"
"Segment"
"Setting"
"Table"
"Timeline"])
+
+(def inlined-models
+ "An additional list of models which are inlined into parent entities for serialization.
+ These are not extracted and serialized separately, but they may need some processing done.
+ For example, the models should also have their entity_id fields populated (if they have one)."
+ ["DashboardCard"
+ "Dimension"
+ "TimelineEvent"])
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/storage/yaml.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/storage/yaml.clj
index cb4aa7423385b1d30162f96c9b1a1bd571aa1858..27214a6197d2e491db73cce31b261952fb127277 100644
--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/storage/yaml.clj
+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/storage/yaml.clj
@@ -22,8 +22,8 @@
(io/make-parents file)
(spit (io/file file) (generate-yaml obj)))
-(defn- store-entity! [{:keys [root-dir]} entity]
- (spit-yaml (u.yaml/hierarchy->file root-dir (serdes.base/serdes-path entity))
+(defn- store-entity! [opts entity]
+ (spit-yaml (u.yaml/hierarchy->file opts entity)
(dissoc entity :serdes/meta)))
(defn- store-settings! [{:keys [root-dir]} settings]
@@ -37,7 +37,8 @@
(instance? java.io.File (:root-dir opts)))
(throw (ex-info ":yaml storage requires the :root-dir option to be a string or File"
{:opts opts})))
- (let [settings (atom [])]
+ (let [settings (atom [])
+ opts (merge opts (serdes.base/storage-base-context))]
(doseq [entity stream]
(if (-> entity :serdes/meta last :model (= "Setting"))
(swap! settings conj entity)
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/utils/yaml.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/utils/yaml.clj
index f40b6dc795e14777683951c93af3a057a55b87a7..6b89fb1e22e3d35af2cc86cb4e1444ecfbd54d20 100644
--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/utils/yaml.clj
+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/utils/yaml.clj
@@ -1,24 +1,10 @@
(ns metabase-enterprise.serialization.v2.utils.yaml
(:require [clojure.java.io :as io]
- [clojure.string :as str])
+ [clojure.string :as str]
+ [metabase.models.serialization.base :as serdes.base])
(:import java.io.File
java.nio.file.Path))
-(def ^:private max-label-length 100)
-
-(defn- truncate-label [s]
- (if (> (count s) max-label-length)
- (subs s 0 max-label-length)
- s))
-
-(defn- leaf-file-name
- ([id] (str id ".yaml"))
- ;; + is a legal, unescaped character on all common filesystems,
- ;; but doesn't appear in `identity-hash` or NanoID!
- ([id label] (if (nil? label)
- (leaf-file-name id)
- (str id "+" (truncate-label label) ".yaml"))))
-
(defn- escape-segment
"Given a path segment, which is supposed to be the name of a single file or directory, escape any slashes inside it.
This occurs in practice, for example with a `Field.name` containing a slash like \"Company/organization website\"."
@@ -35,26 +21,14 @@
(str/replace "__BACKSLASH__" "\\")))
(defn hierarchy->file
- "Given a :serdes/meta abstract path, return a [[File]] corresponding to it."
- ^File [root-dir hierarchy]
- (let [;; All earlier parts of the hierarchy form Model/id/ pairs.
- prefix (apply concat (for [{:keys [model id]} (drop-last hierarchy)]
- [model id]))
- ;; The last part of the hierarchy is used for the basename; this is the only part with the label.
- {:keys [id model label]} (last hierarchy)
- leaf-name (leaf-file-name id label)
- as-given (apply io/file root-dir (map escape-segment (concat prefix [model leaf-name])))]
- (if (.exists ^File as-given)
- as-given
- ; If that file name doesn't exist, check the directory to see if there's one that's the requested file plus a
- ; human-readable portion.
- (let [dir (apply io/file root-dir (map escape-segment (concat prefix [model])))
- matches (filter #(and (.startsWith ^String % (str id "+"))
- (.endsWith ^String % ".yaml"))
- (.list ^File dir))]
- (if (empty? matches)
- (io/file dir (escape-segment leaf-name))
- (io/file dir (first matches)))))))
+ "Given an extracted entity, return a [[File]] corresponding to it."
+ ^File [ctx entity]
+ (let [;; Get the desired [[serdes.base/storage-path]].
+ base-path (serdes.base/storage-path entity ctx)
+ dirnames (drop-last base-path)
+ ;; Attach the file extension to the last part.
+ basename (str (last base-path) ".yaml")]
+ (apply io/file (:root-dir ctx) (map escape-segment (concat dirnames [basename])))))
(defn path-split
"Given a root directory and a file underneath it, return a sequence of path parts to get there.
@@ -67,13 +41,16 @@
(defn path->hierarchy
"Given the list of file path chunks as returned by [[path-split]], reconstruct the `:serdes/meta` abstract path
corresponding to it.
- Note that the __SLASH__ and __BACKSLASH__ interpolations of [[escape-segment]] are reversed here."
+ Note that the __SLASH__ and __BACKSLASH__ interpolations of [[escape-segment]] are reversed here, and also the
+ file extension is stripped off the last segment.
+
+ The heavy lifting is done by the matcher functions registered by each model using
+ [[serdes.base/register-ingestion-path!]]."
[path-parts]
- (let [parentage (into [] (for [[model id] (partition 2 (drop-last 2 path-parts))]
- {:model model :id (unescape-segment id)}))
- [model basename] (take-last 2 path-parts)
- basename (unescape-segment basename)
- [_ id label] (or (re-matches #"^([A-Za-z0-9_\.:-]+)(?:\+(.*))?\.yaml$" basename)
- (re-matches #"^(.+)\.yaml$" basename))]
- (conj parentage (cond-> {:model model :id id}
- label (assoc :label label)))))
+ (let [basename (last path-parts)
+ basename (if (str/ends-with? basename ".yaml")
+ (subs basename 0 (- (count basename) 5))
+ basename)
+ path-parts (concat (map unescape-segment (drop-last path-parts))
+ [(unescape-segment basename)])]
+ (serdes.base/ingest-path path-parts)))
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/api/serialize_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/api/serialize_test.clj
index 17b771b2a23b4f295dd21d9335c95a8b1cb28ccc..b45ee49e713219050404ff0607f7ed67190a016e 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/api/serialize_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/api/serialize_test.clj
@@ -10,7 +10,9 @@
(defn- do-serialize-data-model [f]
(premium-features-test/with-premium-features #{:serialization}
- (mt/with-temp* [Collection [{collection-id :id}]
+ (mt/with-temp* [Collection [{collection-id :id
+ collection-eid :entity_id
+ collection-slug :slug}]
Dashboard [{dashboard-id :id} {:collection_id collection-id}]
Card [{card-id :id} {:collection_id collection-id}]
DashboardCard [_ {:card_id card-id, :dashboard_id dashboard-id}]]
@@ -19,11 +21,15 @@
(is (= collection-id
(db/select-one-field :collection_id Card :id card-id))))
(mt/with-temp-dir [dir "serdes-dir"]
- (f {:collection-id collection-id, :dir dir})))))
+ (f {:collection-id collection-id
+ :collection-filename (if collection-slug
+ (str collection-eid "_" collection-slug)
+ collection-eid)
+ :dir dir})))))
(deftest serialize-data-model-happy-path-test
(do-serialize-data-model
- (fn [{:keys [collection-id dir]}]
+ (fn [{:keys [collection-id collection-filename dir]}]
(is (= {:status "ok"}
(mt/user-http-request :crowberto :post 200 "ee/serialization/serialize/data-model"
{:collection_ids [collection-id]
@@ -35,18 +41,18 @@
(path-files (apply u.files/get-path dir path-components)))]
(is (= (map
#(.toString (u.files/get-path (System/getProperty "java.io.tmpdir") "serdes-dir" %))
- ["Card" "Collection" "Dashboard" "settings.yaml"])
+ ["collections" "settings.yaml"])
(files)))
(testing "subdirs"
- (testing "Card"
+ (testing "cards"
(is (= 1
- (count (files "Card")))))
- (testing "Collection"
+ (count (files "collections" collection-filename "cards")))))
+ (testing "collections"
(is (= 1
- (count (files "Collection")))))
- (testing "Dashboard"
+ (count (remove #{"cards" "dashboards" "timelines"} (files "collections"))))))
+ (testing "dashboards"
(is (= 1
- (count (files "Dashboard")))))))))))
+ (count (files "collections" collection-filename "dashboards")))))))))))
(deftest serialize-data-model-validation-test
(do-serialize-data-model
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e/yaml_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e/yaml_test.clj
index 5df75c314bb44a08e8742e9b497bb10813acfe2f..5decc7568ad7750da38ce927688ecc3d90b91ff0 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e/yaml_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e/yaml_test.clj
@@ -13,18 +13,39 @@
[toucan.db :as db]
[yaml.core :as yaml]))
-(defn- dir->file-set [dir]
+(defn- dir->contents-set [p dir]
(->> dir
.listFiles
- (filter #(.isFile %))
+ (filter p)
(map #(.getName %))
set))
+(defn- dir->file-set [dir]
+ (dir->contents-set #(.isFile %) dir))
+
+(defn- dir->dir-set [dir]
+ (dir->contents-set #(.isDirectory %) dir))
+
(defn- subdirs [dir]
(->> dir
.listFiles
(remove #(.isFile %))))
+(defn- collections [dir]
+ (for [coll-dir (subdirs dir)
+ :when (->> ["cards" "dashboards" "timelines"]
+ (map #(io/file coll-dir %))
+ (filter #(= % coll-dir))
+ empty?)]
+ coll-dir))
+
+(defn- file-set [dir]
+ (let [base (.toPath dir)]
+ (set (for [file (file-seq dir)
+ :when (.isFile file)
+ :let [rel (.relativize base (.toPath file))]]
+ (mapv str rel)))))
+
(defn- random-keyword
([prefix n] (random-keyword prefix n 0))
([prefix n floor] (keyword (str (name prefix) (+ floor (rand-int n))))))
@@ -63,11 +84,12 @@
(let [extraction (atom nil)
entities (atom nil)]
(ts/with-source-and-dest-dbs
- ;; TODO Generating some nested collections would make these tests more robust.
+ ;; TODO Generating some nested collections would make these tests more robust, but that's difficult.
+ ;; There are handwritten tests for storage and ingestion that check out the nesting, at least.
(ts/with-source-db
(testing "insert"
(test-gen/insert!
- {:collection [[100 {:refs {:personal_owner_id ::rs/omit}}]
+ {:collection [[100 {:refs {:personal_owner_id ::rs/omit}}]
[10 {:refs {:personal_owner_id ::rs/omit}
:spec-gen {:namespace :snippets}}]]
:database [[10]]
@@ -145,73 +167,88 @@
(update m (-> entity :serdes/meta last :model)
(fnil conj []) entity))
{} @extraction))
- (is (= 110 (-> @entities (get "Collection") count)))))
-
- (testing "storage"
- (storage.yaml/store! (seq @extraction) dump-dir)
-
- (testing "for Collections"
- (is (= 110 (count (dir->file-set (io/file dump-dir "Collection"))))))
-
- (testing "for Databases"
- (is (= 10 (count (dir->file-set (io/file dump-dir "Database"))))))
-
- (testing "for Tables"
- (is (= 100
- (reduce + (for [db (get @entities "Database")
- :let [tables (dir->file-set (io/file dump-dir "Database" (:name db) "Table"))]]
- (count tables))))
- "Tables are scattered, so the directories are harder to count"))
-
- (testing "for Fields"
- (is (= 1000
- (reduce + (for [db (get @entities "Database")
- table (subdirs (io/file dump-dir "Database" (:name db) "Table"))]
- (->> (io/file table "Field")
- dir->file-set
- count))))
- "Fields are scattered, so the directories are harder to count"))
-
- (testing "for cards"
- (is (= 100 (count (dir->file-set (io/file dump-dir "Card"))))))
-
- (testing "for dashboards"
- (is (= 100 (count (dir->file-set (io/file dump-dir "Dashboard"))))))
-
- (testing "for metrics"
- (is (= 30 (count (dir->file-set (io/file dump-dir "Metric"))))))
-
- (testing "for segments"
- (is (= 30 (count (dir->file-set (io/file dump-dir "Segment"))))))
-
- (testing "for pulses"
- (is (= 30 (count (dir->file-set (io/file dump-dir "Pulse"))))))
-
- (testing "for pulse cards"
- (is (= 120 (reduce + (for [pulse (get @entities "Pulse")]
- (->> (io/file dump-dir "Pulse" (:entity_id pulse) "PulseCard")
- dir->file-set
- count))))))
-
- (testing "for pulse channels"
- (is (= 30 (reduce + (for [pulse (get @entities "Pulse")]
- (->> (io/file dump-dir "Pulse" (:entity_id pulse) "PulseChannel")
- dir->file-set
- count)))))
- (is (= 40 (reduce + (for [{:keys [recipients]} (get @entities "PulseChannel")]
- (count recipients))))))
-
- (testing "for native query snippets"
- (is (= 10 (count (dir->file-set (io/file dump-dir "NativeQuerySnippet"))))))
-
- (testing "for timelines and events"
- (is (= 10 (count (dir->file-set (io/file dump-dir "Timeline"))))))
-
- (testing "for settings"
- (is (.exists (io/file dump-dir "settings.yaml")))))
+ (is (= 110 (-> @entities (get "Collection") count))))
+
+ (testing "storage"
+ (storage.yaml/store! (seq @extraction) dump-dir)
+
+ (testing "for Collections"
+ (is (= 110 (count (for [f (file-set (io/file dump-dir))
+ :when (and (= (first f) "collections")
+ (let [[a b] (take-last 2 f)]
+ (= b (str a ".yaml"))))]
+ f)))
+ "which all go in collections/, even the snippets ones"))
+
+ (testing "for Databases"
+ (is (= 10 (count (dir->dir-set (io/file dump-dir "databases"))))))
+
+ (testing "for Tables"
+ (is (= 100
+ (reduce + (for [db (get @entities "Database")
+ :let [tables (dir->dir-set (io/file dump-dir "databases" (:name db) "tables"))]]
+ (count tables))))
+ "Tables are scattered, so the directories are harder to count"))
+
+ (testing "for Fields"
+ (is (= 1000
+ (reduce + (for [db (get @entities "Database")
+ table (subdirs (io/file dump-dir "databases" (:name db) "tables"))]
+ (->> (io/file table "fields")
+ dir->file-set
+ count))))
+ "Fields are scattered, so the directories are harder to count"))
+
+ (testing "for cards"
+ (is (= 100 (->> (io/file dump-dir "collections")
+ collections
+ (map (comp count dir->file-set #(io/file % "cards")))
+ (reduce +)))))
+
+ (testing "for dashboards"
+ (is (= 100 (->> (io/file dump-dir "collections")
+ collections
+ (map (comp count dir->file-set #(io/file % "dashboards")))
+ (reduce +)))))
+
+ (testing "for timelines"
+ (is (= 10 (->> (io/file dump-dir "collections")
+ collections
+ (map (comp count dir->file-set #(io/file % "timelines")))
+ (reduce +)))))
+
+ (testing "for metrics"
+ (is (= 30 (reduce + (for [db (dir->dir-set (io/file dump-dir "databases"))
+ table (dir->dir-set (io/file dump-dir "databases" db "tables"))
+ :let [metrics-dir (io/file dump-dir "databases" db "tables" table "metrics")]
+ :when (.exists metrics-dir)]
+ (count (dir->file-set metrics-dir)))))))
+
+ (testing "for segments"
+ (is (= 30 (reduce + (for [db (dir->dir-set (io/file dump-dir "databases"))
+ table (dir->dir-set (io/file dump-dir "databases" db "tables"))
+ :let [segments-dir (io/file dump-dir "databases" db "tables" table "segments")]
+ :when (.exists segments-dir)]
+ (count (dir->file-set segments-dir)))))))
+
+ (testing "for native query snippets"
+ (is (= 10 (->> (io/file dump-dir "snippets")
+ collections
+ (map (comp count dir->file-set))
+ (reduce +)))))
+
+ (testing "for settings"
+ (is (.exists (io/file dump-dir "settings.yaml"))))))
(testing "ingest and load"
(ts/with-dest-db
+ (testing "ingested set matches extracted set"
+ (let [extracted-set (set (map (comp #'ingest.yaml/strip-labels serdes.base/serdes-path) @extraction))]
+ (is (= (count extracted-set)
+ (count @extraction)))
+ (is (= extracted-set
+ (set (keys (#'ingest.yaml/ingest-all (io/file dump-dir))))))))
+
(testing "doing ingestion"
(is (serdes.load/load-metabase (ingest.yaml/ingest-yaml dump-dir))
"successful"))
@@ -290,32 +327,6 @@
(serdes.base/extract-one "Segment" {})
clean-entity)))))
- (testing "for pulses"
- (doseq [{:keys [entity_id] :as pulse} (get @entities "Pulse")]
- (is (= (clean-entity pulse)
- (->> (db/select-one 'Pulse :entity_id entity_id)
- (serdes.base/extract-one "Pulse" {})
- clean-entity)))))
-
- (testing "for pulse cards"
- (doseq [{:keys [entity_id] :as card} (get @entities "PulseCard")]
- (is (= (clean-entity card)
- (->> (db/select-one 'PulseCard :entity_id entity_id)
- (serdes.base/extract-one "PulseCard" {})
- clean-entity)))))
-
- (testing "for pulse channels"
- (doseq [{:keys [entity_id] :as channel} (get @entities "PulseChannel")]
- ;; The :recipients list is in arbitrary order - turn them into sets for comparison.
- (is (= (-> channel
- (update :recipients set)
- clean-entity)
- (let [loaded-channel (->> (db/select-one 'PulseChannel :entity_id entity_id)
- (serdes.base/extract-one "PulseChannel" {}))]
- (-> loaded-channel
- (update :recipients set)
- clean-entity))))))
-
(testing "for native query snippets"
(doseq [{:keys [entity_id] :as snippet} (get @entities "NativeQuerySnippet")]
(is (= (clean-entity snippet)
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/ingest/yaml_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/ingest/yaml_test.clj
index e72c04a76ff2d5b1fe7beb7e1c6ccec665a80cc7..02f0e40c7a42f8d7a186a660a075bd70e99a39e0 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/ingest/yaml_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/ingest/yaml_test.clj
@@ -8,28 +8,32 @@
(deftest basic-ingest-test
(ts/with-random-dump-dir [dump-dir "serdesv2-"]
- (io/make-parents dump-dir "Collection" "fake") ; Prepare the right directories.
+ (io/make-parents dump-dir "collections" "1234567890abcdefABCDE_the_label" "fake") ; Prepare the right directories.
+ (io/make-parents dump-dir "collections" "0987654321zyxwvuABCDE" "fake")
+
(spit (io/file dump-dir "settings.yaml")
(yaml/generate-string {:some-key "with string value"
:another-key 7
:blank-key nil}))
- (spit (io/file dump-dir "Collection" "fake-id+the_label.yaml")
- (yaml/generate-string {:some "made up" :data "here" :entity_id "fake-id" :slug "the_label"}))
- (spit (io/file dump-dir "Collection" "no-label.yaml")
- (yaml/generate-string {:some "other" :data "in this one" :entity_id "no-label"}))
+ (spit (io/file dump-dir "collections" "1234567890abcdefABCDE_the_label" "1234567890abcdefABCDE_the_label.yaml")
+ (yaml/generate-string {:some "made up" :data "here" :entity_id "1234567890abcdefABCDE" :slug "the_label"}))
+ (spit (io/file dump-dir "collections" "0987654321zyxwvuABCDE" "0987654321zyxwvuABCDE.yaml")
+ (yaml/generate-string {:some "other" :data "in this one" :entity_id "0987654321zyxwvuABCDE"}))
(let [ingestable (ingest.yaml/ingest-yaml dump-dir)
- exp-files {[{:model "Collection" :id "fake-id" :label "the_label"}] {:some "made up"
- :data "here"
- :entity_id "fake-id"
- :slug "the_label"}
- [{:model "Collection" :id "no-label"}] {:some "other"
- :data "in this one"
- :entity_id "no-label"}
- [{:model "Setting" :id "some-key"}] {:key :some-key :value "with string value"}
- [{:model "Setting" :id "another-key"}] {:key :another-key :value 7}
- [{:model "Setting" :id "blank-key"}] {:key :blank-key :value nil}}]
- (testing "the right set of file is returned by ingest-list"
+ exp-files {[{:model "Collection"
+ :id "1234567890abcdefABCDE"
+ :label "the_label"}] {:some "made up"
+ :data "here"
+ :entity_id "1234567890abcdefABCDE"
+ :slug "the_label"}
+ [{:model "Collection" :id "0987654321zyxwvuABCDE"}] {:some "other"
+ :data "in this one"
+ :entity_id "0987654321zyxwvuABCDE"}
+ [{:model "Setting" :id "some-key"}] {:key :some-key :value "with string value"}
+ [{:model "Setting" :id "another-key"}] {:key :another-key :value 7}
+ [{:model "Setting" :id "blank-key"}] {:key :blank-key :value nil}}]
+ (testing "the right set of files is returned by ingest-list"
(is (= (set (keys exp-files))
(into #{} (ingest/ingest-list ingestable)))))
@@ -46,29 +50,30 @@
(deftest flexible-file-matching-test
(ts/with-random-dump-dir [dump-dir "serdesv2-"]
- (io/make-parents dump-dir "Collection" "fake")
- (spit (io/file dump-dir "Collection" "entity-id+human-readable-things.yaml")
+ (io/make-parents dump-dir "collections" "1234567890abcdefABCDE_human-readable-things" "fake")
+ (spit (io/file dump-dir "collections" "1234567890abcdefABCDE_human-readable-things"
+ "1234567890abcdefABCDE_human-readable-things.yaml")
(yaml/generate-string {:some "made up" :data "here"}))
(let [ingestable (ingest.yaml/ingest-yaml dump-dir)
exp {:some "made up"
:data "here"
- :serdes/meta [{:model "Collection" :id "entity-id"}]}]
+ :serdes/meta [{:model "Collection" :id "1234567890abcdefABCDE"}]}]
(testing "the returned set of files has the human-readable labels"
- (is (= #{[{:model "Collection" :id "entity-id" :label "human-readable-things"}]}
+ (is (= #{[{:model "Collection" :id "1234567890abcdefABCDE" :label "human-readable-things"}]}
(into #{} (ingest/ingest-list ingestable)))))
(testing "fetching the file with the label works"
(is (= exp
- (ingest/ingest-one ingestable [{:model "Collection" :id "entity-id" :label "human-readable-things"}]))))
+ (ingest/ingest-one ingestable [{:model "Collection" :id "1234567890abcdefABCDE" :label "human-readable-things"}]))))
(testing "fetching the file without the label also works"
(is (= exp
- (ingest/ingest-one ingestable [{:model "Collection" :id "entity-id"}])))))))
+ (ingest/ingest-one ingestable [{:model "Collection" :id "1234567890abcdefABCDE"}])))))))
(deftest file-name-escaping-test
(ts/with-random-dump-dir [dump-dir "serdesv2-"]
- (io/make-parents dump-dir "Database" "fake")
- (spit (io/file dump-dir "Database" "my data__SLASH__speculations.yaml")
+ (io/make-parents dump-dir "databases" "my data__SLASH__speculations" "fake")
+ (spit (io/file dump-dir "databases" "my data__SLASH__speculations" "my data__SLASH__speculations.yaml")
(yaml/generate-string {:some "made up" :data "here"}))
(let [ingestable (ingest.yaml/ingest-yaml dump-dir)
@@ -85,14 +90,14 @@
(deftest keyword-reconstruction-test
(ts/with-random-dump-dir [dump-dir "serdesv2-"]
- (io/make-parents dump-dir "Card" "fake")
- (spit (io/file dump-dir "Card" "some-card.yaml")
+ (io/make-parents dump-dir "collections" "cards" "fake")
+ (spit (io/file dump-dir "collections" "cards" "1234567890abcdefABCDE_some_card.yaml")
(yaml/generate-string {:visualization_settings
{:column_settings {"[\"name\",\"sum\"]" {:number_style "currency"}}}}))
(let [ingestable (ingest.yaml/ingest-yaml dump-dir)
exp {:visualization_settings {:column_settings {"[\"name\",\"sum\"]" {:number_style "currency"}}}
- :serdes/meta [{:model "Card" :id "some-card"}]}]
+ :serdes/meta [{:model "Card" :id "1234567890abcdefABCDE"}]}]
(testing "the file as read in correctly reconstructs keywords only where legal"
(is (= exp
- (ingest/ingest-one ingestable [{:model "Card" :id "some-card"}])))))))
+ (ingest/ingest-one ingestable [{:model "Card" :id "1234567890abcdefABCDE"}])))))))
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj
index 78453c2135638d2dd7de7fd6a684d388486f0422..340c631ef35f9e6f7d1e0e41b5c91681d4a1aa1a 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj
@@ -5,8 +5,8 @@
[metabase-enterprise.serialization.v2.extract :as serdes.extract]
[metabase-enterprise.serialization.v2.ingest :as serdes.ingest]
[metabase-enterprise.serialization.v2.load :as serdes.load]
- [metabase.models :refer [Card Collection Dashboard DashboardCard Database Field FieldValues Metric Pulse
- PulseChannel PulseChannelRecipient Segment Table Timeline TimelineEvent User]]
+ [metabase.models :refer [Card Collection Dashboard DashboardCard Database Field FieldValues Metric
+ Segment Table Timeline TimelineEvent User]]
[metabase.models.serialization.base :as serdes.base]
[schema.core :as s]
[toucan.db :as db])
@@ -163,76 +163,6 @@
(is (db/exists? Table :name "posts" :db_id (:id @db1d)))
(is (db/exists? Table :name "posts" :db_id (:id @db2d)))))))))
-(deftest pulse-channel-recipient-merging-test
- (testing "pulse channel recipients are listed as emails on a channel, then merged with the existing ones"
- (let [serialized (atom nil)
- u1s (atom nil)
- u2s (atom nil)
- u3s (atom nil)
- pulse-s (atom nil)
- pc1s (atom nil)
- pc2s (atom nil)
- pcr1s (atom nil)
- pcr2s (atom nil)
-
- u1d (atom nil)
- u2d (atom nil)
- u3d (atom nil)
- pulse-d (atom nil)
- pc1d (atom nil)]
- (ts/with-source-and-dest-dbs
- (testing "serializing the pulse, channel and recipients"
- (ts/with-source-db
- (reset! u1s (ts/create! User :first_name "Alex" :last_name "Lifeson" :email "alifeson@rush.yyz"))
- (reset! u2s (ts/create! User :first_name "Geddy" :last_name "Lee" :email "glee@rush.yyz"))
- (reset! u3s (ts/create! User :first_name "Neil" :last_name "Peart" :email "neil@rush.yyz"))
- (reset! pulse-s (ts/create! Pulse :name "Heartbeat" :creator_id (:id @u1s)))
- (reset! pc1s (ts/create! PulseChannel
- :pulse_id (:id @pulse-s)
- :channel_type :email
- :schedule_type :daily
- :schedule_hour 16))
- (reset! pc2s (ts/create! PulseChannel
- :pulse_id (:id @pulse-s)
- :channel_type :slack
- :schedule_type :hourly))
- ;; Only Lifeson and Lee are recipients in the source.
- (reset! pcr1s (ts/create! PulseChannelRecipient :pulse_channel_id (:id @pc1s) :user_id (:id @u1s)))
- (reset! pcr2s (ts/create! PulseChannelRecipient :pulse_channel_id (:id @pc1s) :user_id (:id @u2s)))
- (reset! serialized (into [] (serdes.extract/extract-metabase {})))))
-
- (testing "recipients are serialized as :recipients [email] on the PulseChannel"
- (is (= #{["alifeson@rush.yyz" "glee@rush.yyz"]
- []}
- (set (map :recipients (by-model @serialized "PulseChannel"))))))
-
- (testing "deserialization merges the existing recipients with the new ones"
- (ts/with-dest-db
- ;; Users in a different order, so different IDs.
- (reset! u2d (ts/create! User :first_name "Geddy" :last_name "Lee" :email "glee@rush.yyz"))
- (reset! u1d (ts/create! User :first_name "Alex" :last_name "Lifeson" :email "alifeson@rush.yyz"))
- (reset! u3d (ts/create! User :first_name "Neil" :last_name "Peart" :email "neil@rush.yyz"))
- (reset! pulse-d (ts/create! Pulse :name "Heartbeat" :creator_id (:id @u1d) :entity_id (:entity_id @pulse-s)))
- (reset! pc1d (ts/create! PulseChannel
- :entity_id (:entity_id @pc1s)
- :pulse_id (:id @pulse-d)
- :channel_type :email
- :schedule_type :daily
- :schedule_hour 16))
- ;; Only Lee and Peart are recipients in the source.
- (ts/create! PulseChannelRecipient :pulse_channel_id (:id @pc1d) :user_id (:id @u2d))
- (ts/create! PulseChannelRecipient :pulse_channel_id (:id @pc1d) :user_id (:id @u3d))
-
- (is (= 2 (db/count PulseChannelRecipient)))
- (is (= #{(:id @u2d) (:id @u3d)}
- (db/select-field :user_id PulseChannelRecipient)))
-
- (serdes.load/load-metabase (ingestion-in-memory @serialized))
-
- (is (= 3 (db/count PulseChannelRecipient)))
- (is (= #{(:id @u1d) (:id @u2d) (:id @u3d)}
- (db/select-field :user_id PulseChannelRecipient)))))))))
-
(deftest card-dataset-query-test
;; Card.dataset_query is a JSON-encoded MBQL query, which contain database, table, and field IDs - these need to be
;; converted to a portable form and read back in.
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/storage/yaml_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/storage/yaml_test.clj
index 8542a969b36c023910f628314bc7a02dcf3b3e65..9b6b3560a7a97fe8b5e9fe375e1d980e09f81316 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/storage/yaml_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/storage/yaml_test.clj
@@ -6,18 +6,18 @@
[metabase-enterprise.serialization.test-util :as ts]
[metabase-enterprise.serialization.v2.extract :as extract]
[metabase-enterprise.serialization.v2.storage.yaml :as storage.yaml]
- [metabase.models :refer [Collection Database Field FieldValues Table]]
+ [metabase.models :refer [Card Collection Dashboard Database Field FieldValues NativeQuerySnippet Table]]
[metabase.models.serialization.base :as serdes.base]
[metabase.util.date-2 :as u.date]
[toucan.db :as db]
[yaml.core :as yaml]))
-(defn- dir->file-set [dir]
- (->> dir
- .listFiles
- (filter #(.isFile %))
- (map #(.getName %))
- set))
+(defn- file-set [dir]
+ (let [base (.toPath dir)]
+ (set (for [file (file-seq dir)
+ :when (.isFile file)
+ :let [rel (.relativize base (.toPath file))]]
+ (mapv str rel)))))
(deftest basic-dump-test
(ts/with-random-dump-dir [dump-dir "serdesv2-"]
@@ -25,15 +25,16 @@
(ts/with-temp-dpc [Collection [parent {:name "Some Collection"}]
Collection [child {:name "Child Collection" :location (format "/%d/" (:id parent))}]]
(let [export (into [] (extract/extract-metabase nil))
- parent-filename (format "%s+some_collection.yaml" (:entity_id parent))
- child-filename (format "%s+child_collection.yaml" (:entity_id child))]
+ parent-filename (format "%s_some_collection" (:entity_id parent))
+ child-filename (format "%s_child_collection" (:entity_id child))]
(storage.yaml/store! export dump-dir)
(testing "the right files in the right places"
- (is (= #{parent-filename child-filename}
- (dir->file-set (io/file dump-dir "Collection")))
- "Entities go in subdirectories")
- (is (= #{"settings.yaml"}
- (dir->file-set (io/file dump-dir)))
+ (is (= #{[parent-filename (str parent-filename ".yaml")]
+ [parent-filename child-filename (str child-filename ".yaml")]}
+ (file-set (io/file dump-dir "collections")))
+ "collections form a tree, with same-named files")
+ (is (contains? (file-set (io/file dump-dir))
+ ["settings.yaml"])
"A few top-level files are expected"))
(testing "the Collections properly exported"
@@ -41,42 +42,107 @@
(dissoc :id :location)
(assoc :parent_id nil)
(update :created_at t/offset-date-time))
- (-> (yaml/from-file (io/file dump-dir "Collection" parent-filename))
+ (-> (yaml/from-file (io/file dump-dir "collections" parent-filename (str parent-filename ".yaml")))
(update :created_at t/offset-date-time))))
(is (= (-> (into {} (db/select-one Collection :id (:id child)))
(dissoc :id :location)
(assoc :parent_id (:entity_id parent))
(update :created_at t/offset-date-time))
- (-> (yaml/from-file (io/file dump-dir "Collection" child-filename))
+ (-> (yaml/from-file (io/file dump-dir "collections" parent-filename
+ child-filename (str child-filename ".yaml")))
(update :created_at t/offset-date-time))))))))))
+(deftest collection-nesting-test
+ (ts/with-random-dump-dir [dump-dir "serdesv2-"]
+ (ts/with-empty-h2-app-db
+ (ts/with-temp-dpc [Collection [grandparent {:name "Grandparent Collection"
+ :location "/"}]
+ Collection [parent {:name "Parent Collection"
+ :location (str "/" (:id grandparent) "/")}]
+ Collection [child {:name "Child Collection"
+ :location (str "/" (:id grandparent) "/" (:id parent) "/")}]
+ Card [c1 {:name "root card" :collection_id nil}]
+ Card [c2 {:name "grandparent card" :collection_id (:id grandparent)}]
+ Card [c3 {:name "parent card" :collection_id (:id parent)}]
+ Card [c4 {:name "child card" :collection_id (:id child)}]
+ Dashboard [d1 {:name "parent dash" :collection_id (:id parent)}]]
+ (let [export (into [] (extract/extract-metabase nil))]
+ (storage.yaml/store! export dump-dir)
+ (testing "the right files in the right places"
+ (let [gp-dir (str (:entity_id grandparent) "_grandparent_collection")
+ p-dir (str (:entity_id parent) "_parent_collection")
+ c-dir (str (:entity_id child) "_child_collection")]
+ (is (= #{[gp-dir (str gp-dir ".yaml")] ; Grandparent collection
+ [gp-dir p-dir (str p-dir ".yaml")] ; Parent collection
+ [gp-dir p-dir c-dir (str c-dir ".yaml")] ; Child collection
+ ["cards" (str (:entity_id c1) "_root_card.yaml")] ; Root card
+ [gp-dir "cards" (str (:entity_id c2) "_grandparent_card.yaml")] ; Grandparent card
+ [gp-dir p-dir "cards" (str (:entity_id c3) "_parent_card.yaml")] ; Parent card
+ [gp-dir p-dir c-dir "cards" (str (:entity_id c4) "_child_card.yaml")] ; Child card
+ [gp-dir p-dir "dashboards" (str (:entity_id d1) "_parent_dash.yaml")]} ; Parent dashboard
+ (file-set (io/file dump-dir "collections")))))))))))
+
+(deftest snippets-collections-nesting-test
+ (ts/with-random-dump-dir [dump-dir "serdesv2-"]
+ (ts/with-empty-h2-app-db
+ (ts/with-temp-dpc [Collection [grandparent {:name "Grandparent Collection"
+ :namespace :snippets
+ :location "/"}]
+ Collection [parent {:name "Parent Collection"
+ :namespace :snippets
+ :location (str "/" (:id grandparent) "/")}]
+ Collection [child {:name "Child Collection"
+ :namespace :snippets
+ :location (str "/" (:id grandparent) "/" (:id parent) "/")}]
+ NativeQuerySnippet [c1 {:name "root snippet" :collection_id nil}]
+ NativeQuerySnippet [c2 {:name "grandparent snippet" :collection_id (:id grandparent)}]
+ NativeQuerySnippet [c3 {:name "parent snippet" :collection_id (:id parent)}]
+ NativeQuerySnippet [c4 {:name "child snippet" :collection_id (:id child)}]]
+ (let [export (into [] (extract/extract-metabase nil))]
+ (storage.yaml/store! export dump-dir)
+ (let [gp-dir (str (:entity_id grandparent) "_grandparent_collection")
+ p-dir (str (:entity_id parent) "_parent_collection")
+ c-dir (str (:entity_id child) "_child_collection")]
+ (testing "collections under collections/"
+ (is (= #{[gp-dir (str gp-dir ".yaml")] ; Grandparent collection
+ [gp-dir p-dir (str p-dir ".yaml")] ; Parent collection
+ [gp-dir p-dir c-dir (str c-dir ".yaml")]} ; Child collection
+ (file-set (io/file dump-dir "collections")))))
+ (testing "snippets under snippets/"
+ (is (= #{
+ [(str (:entity_id c1) "_root_snippet.yaml")] ; Root snippet
+ [gp-dir (str (:entity_id c2) "_grandparent_snippet.yaml")] ; Grandparent snippet
+ [gp-dir p-dir (str (:entity_id c3) "_parent_snippet.yaml")] ; Parent snippet
+ [gp-dir p-dir c-dir (str (:entity_id c4) "_child_snippet.yaml")]} ; Child snippet
+ (file-set (io/file dump-dir "snippets")))))))))))
+
(deftest embedded-slash-test
(ts/with-random-dump-dir [dump-dir "serdesv2-"]
(ts/with-empty-h2-app-db
(ts/with-temp-dpc [Database [db {:name "My Company Data"}]
Table [table {:name "Customers" :db_id (:id db)}]
Field [website {:name "Company/organization website" :table_id (:id table)}]
- FieldValues [_ {:field_id (:id website)}]]
+ FieldValues [_ {:field_id (:id website)}]
+ Table [_ {:name "Orders/Invoices" :db_id (:id db)}]]
(let [export (into [] (extract/extract-metabase nil))]
(storage.yaml/store! export dump-dir)
(testing "the right files in the right places"
- (is (= #{"Company__SLASH__organization website.yaml"}
- (dir->file-set (io/file dump-dir "Database" "My Company Data" "Table" "Customers" "Field")))
+ (is (= #{["Company__SLASH__organization website.yaml"]
+ ["Company__SLASH__organization website___fieldvalues.yaml"]}
+ (file-set (io/file dump-dir "databases" "My Company Data" "tables" "Customers" "fields")))
"Slashes in file names get escaped")
- (is (= #{"0.yaml"}
- (dir->file-set (io/file dump-dir "Database" "My Company Data" "Table" "Customers"
- "Field" "Company__SLASH__organization website"
- "FieldValues")))
- "Slashes in parent directory names get escaped"))
+ (is (contains? (file-set (io/file dump-dir "databases" "My Company Data" "tables"))
+ ["Orders__SLASH__Invoices" "Orders__SLASH__Invoices.yaml"])
+ "Slashes in directory names get escaped"))
(testing "the Field was properly exported"
(is (= (-> (into {} (serdes.base/extract-one "Field" {} (db/select-one 'Field :id (:id website))))
(update :created_at u.date/format)
(dissoc :serdes/meta))
(-> (yaml/from-file (io/file dump-dir
- "Database" "My Company Data"
- "Table" "Customers"
- "Field" "Company__SLASH__organization website.yaml"))
+ "databases" "My Company Data"
+ "tables" "Customers"
+ "fields" "Company__SLASH__organization website.yaml"))
(update :visibility_type keyword)
(update :base_type keyword))))))))))
diff --git a/src/metabase/models/card.clj b/src/metabase/models/card.clj
index 3fff08e260bd81c44cdb27d89c47571868f60c8e..c1d21b051c642c38f0adedb49d36e0029abba332 100644
--- a/src/metabase/models/card.clj
+++ b/src/metabase/models/card.clj
@@ -332,10 +332,8 @@
[:name (serdes.hash/hydrated-hash :collection "<none>") :created_at])
;;; ------------------------------------------------- Serialization --------------------------------------------------
-(defmethod serdes.base/extract-query "Card" [_ {:keys [collection-set]}]
- (if (seq collection-set)
- (db/select-reducible Card :collection_id [:in collection-set])
- (db/select-reducible Card)))
+(defmethod serdes.base/extract-query "Card" [_ opts]
+ (serdes.base/extract-query-collections Card opts))
(defn- export-result-metadata [metadata]
(when metadata
@@ -366,16 +364,19 @@
;; :table_id and :database_id are extracted as just :table_id [database_name schema table_name].
;; :collection_id is extracted as its entity_id or identity-hash.
;; :creator_id as the user's email.
- (-> (serdes.base/extract-one-basics "Card" card)
- (update :database_id serdes.util/export-fk-keyed 'Database :name)
- (update :table_id serdes.util/export-table-fk)
- (update :collection_id serdes.util/export-fk 'Collection)
- (update :creator_id serdes.util/export-user)
- (update :made_public_by_id serdes.util/export-user)
- (update :dataset_query serdes.util/export-mbql)
- (update :parameter_mappings serdes.util/export-parameter-mappings)
- (update :visualization_settings serdes.util/export-visualization-settings)
- (update :result_metadata export-result-metadata)))
+ (try
+ (-> (serdes.base/extract-one-basics "Card" card)
+ (update :database_id serdes.util/export-fk-keyed 'Database :name)
+ (update :table_id serdes.util/export-table-fk)
+ (update :collection_id serdes.util/export-fk 'Collection)
+ (update :creator_id serdes.util/export-user)
+ (update :made_public_by_id serdes.util/export-user)
+ (update :dataset_query serdes.util/export-mbql)
+ (update :parameter_mappings serdes.util/export-parameter-mappings)
+ (update :visualization_settings serdes.util/export-visualization-settings)
+ (update :result_metadata export-result-metadata))
+ (catch Exception e
+ (throw (ex-info "Failed to export Card" {:card card} e)))))
(defmethod serdes.base/load-xform "Card"
[card]
@@ -415,3 +416,5 @@
(when (seq template-tags)
(set (for [{:keys [card-id]} template-tags]
["Card" card-id]))))))
+
+(serdes.base/register-ingestion-path! "Card" (serdes.base/ingestion-matcher-collected "collections" "Card"))
diff --git a/src/metabase/models/collection.clj b/src/metabase/models/collection.clj
index e8392f57a0695828a308d9258e49accf63177f63..873360f92dccbea3bbe825fc2d61ccbd3bedfefe 100644
--- a/src/metabase/models/collection.clj
+++ b/src/metabase/models/collection.clj
@@ -964,6 +964,21 @@
["Card" card-id]))]
(set/union child-colls dashboards cards)))
+(defmethod serdes.base/storage-path "Collection" [coll {:keys [collections]}]
+ (let [parental (get collections (:entity_id coll))]
+ (concat ["collections"] parental [(last parental)])))
+
+(serdes.base/register-ingestion-path!
+ "Collection"
+ ;; Collections' paths are ["collections" "grandparent" "parent" "me" "me"]
+ (fn [path]
+ (when-let [[id slug] (and (= (first path) "collections")
+ (apply = (take-last 2 path))
+ (serdes.base/split-leaf-file-name (last path)))]
+ (cond-> {:model "Collection" :id id}
+ slug (assoc :label slug)
+ true vector))))
+
;;; +----------------------------------------------------------------------------------------------------------------+
;;; | Perms Checking Helper Fns |
;;; +----------------------------------------------------------------------------------------------------------------+
diff --git a/src/metabase/models/dashboard.clj b/src/metabase/models/dashboard.clj
index bddd277c8fccdf75e9509840a8edee3fb4e27e15..1d4b7249ecb28c1685ca89a7fb4eab83c7492d27 100644
--- a/src/metabase/models/dashboard.clj
+++ b/src/metabase/models/dashboard.clj
@@ -409,11 +409,9 @@
;;; +----------------------------------------------------------------------------------------------------------------+
;;; | SERIALIZATION |
;;; +----------------------------------------------------------------------------------------------------------------+
-(defmethod serdes.base/extract-query "Dashboard" [_ {:keys [collection-set]}]
+(defmethod serdes.base/extract-query "Dashboard" [_ opts]
(eduction (map #(hydrate % :ordered_cards))
- (if (seq collection-set)
- (db/select-reducible Dashboard :collection_id [:in collection-set])
- (db/select-reducible Dashboard))))
+ (serdes.base/extract-query-collections Dashboard opts)))
(defn- extract-dashcard
[dashcard]
@@ -478,3 +476,5 @@
card-id (cond-> (set (keep :card_id parameter_mappings))
card_id (conj card_id))]
["Card" card-id])))
+
+(serdes.base/register-ingestion-path! "Dashboard" (serdes.base/ingestion-matcher-collected "collections" "Dashboard"))
diff --git a/src/metabase/models/dashboard_card.clj b/src/metabase/models/dashboard_card.clj
index 2c5e1a1cc120ec557b771b230973b826550b602e..db333d93892ebe8bf14b3e5090b8d1962aca6022 100644
--- a/src/metabase/models/dashboard_card.clj
+++ b/src/metabase/models/dashboard_card.clj
@@ -10,6 +10,7 @@
[metabase.models.serialization.hash :as serdes.hash]
[metabase.models.serialization.util :as serdes.util]
[metabase.util :as u]
+ [metabase.util.date-2 :as u.date]
[metabase.util.i18n :refer [tru]]
[metabase.util.schema :as su]
[schema.core :as s]
@@ -233,5 +234,6 @@
(dissoc :serdes/meta)
(update :card_id serdes.util/import-fk 'Card)
(update :dashboard_id serdes.util/import-fk 'Dashboard)
+ (update :created_at #(if (string? %) (u.date/parse %) %))
(update :parameter_mappings serdes.util/import-parameter-mappings)
(update :visualization_settings serdes.util/import-visualization-settings)))
diff --git a/src/metabase/models/database.clj b/src/metabase/models/database.clj
index a13708c7721e23ee0a4fe1eda095daf3fa18d347..fe1791586dd14b6ec557425c320f717b075507f8 100644
--- a/src/metabase/models/database.clj
+++ b/src/metabase/models/database.clj
@@ -324,3 +324,16 @@
(m "Database"
(update ingested :details #(or % (:details local) {}))
local)))
+
+(defmethod serdes.base/storage-path "Database" [{:keys [name]} _]
+ ;; ["databases" "db_name" "db_name"] directory for the database with same-named file inside.
+ ["databases" name name])
+
+(serdes.base/register-ingestion-path!
+ "Database"
+ ;; ["databases" "my-db" "my-db"]
+ (fn [[a b c :as path]]
+ (when (and (= (count path) 3)
+ (= a "databases")
+ (= b c))
+ [{:model "Database" :id c}])))
diff --git a/src/metabase/models/field.clj b/src/metabase/models/field.clj
index 53826659ee2564e2be0f3db2bb0ccd1679ebef60..7e3f9270a985f95df440553fb36c62b082af064a 100644
--- a/src/metabase/models/field.clj
+++ b/src/metabase/models/field.clj
@@ -455,3 +455,27 @@
:field_id (:id field)
:serdes/meta [{:model "Dimension" :id (:entity_id dim)}])]
(serdes.base/load-one! dim local)))))
+
+(defmethod serdes.base/storage-path "Field" [field _]
+ (-> field
+ serdes.base/serdes-path
+ drop-last
+ serdes.util/storage-table-path-prefix
+ (concat ["fields" (:name field)])))
+
+(serdes.base/register-ingestion-path!
+ "Field"
+ ;; ["databases" "my-db" "schemas" "PUBLIC" "tables" "customers" "fields" "customer_id"]
+ ;; ["databases" "my-db" "tables" "customers" "fields" "customer_id"]
+ (fn [path]
+ (when-let [{db "databases"
+ schema "schemas"
+ table "tables"
+ field "fields"} (and (#{6 8} (count path))
+ (serdes.base/ingestion-matcher-pairs
+ path [["databases" "schemas" "tables" "fields"]
+ ["databases" "tables" "fields"]]))]
+ (filterv identity [{:model "Database" :id db}
+ (when schema {:model "Schema" :id schema})
+ {:model "Table" :id table}
+ {:model "Field" :id field}]))))
diff --git a/src/metabase/models/field_values.clj b/src/metabase/models/field_values.clj
index 43ad911eb1e6066f01ff6f3414b543e2b472a70c..83bb5e36e43392e8cf34cb3057f02c26be996e77 100644
--- a/src/metabase/models/field_values.clj
+++ b/src/metabase/models/field_values.clj
@@ -16,7 +16,8 @@
sandboxed permission try to get values of a Field.
Normally these FieldValues will be deleted after [[advanced-field-values-max-age]] days by the scanning process.
But they will also be automatically deleted when the Full FieldValues of the same Field got updated."
- (:require [clojure.tools.logging :as log]
+ (:require [clojure.string :as str]
+ [clojure.tools.logging :as log]
[java-time :as t]
[metabase.models.serialization.base :as serdes.base]
[metabase.models.serialization.hash :as serdes.hash]
@@ -456,3 +457,34 @@
(= (:type ingested) (:type local)) (dissoc :type)
(= (:hash_key ingested) (:hash_key local)) (dissoc :hash_key))]
((get-method serdes.base/load-update! "") "FieldValues" ingested local)))
+
+(def ^:private field-values-slug "___fieldvalues")
+
+(defmethod serdes.base/storage-path "FieldValues" [fv _]
+ ;; [path to table "fields" "field-name___fieldvalues"] since there's zero or one FieldValues per Field, and Fields
+ ;; don't have their own directories.
+ (let [hierarchy (serdes.base/serdes-path fv)
+ field (last (drop-last hierarchy))
+ table-prefix (serdes.util/storage-table-path-prefix (drop-last 2 hierarchy))]
+ (concat table-prefix
+ ["fields" (str (:id field) field-values-slug)])))
+
+(serdes.base/register-ingestion-path!
+ "FieldValues"
+ ;; ["databases" "my-db" "schemas" "PUBLIC" "tables" "customers" "fields" "customer_id___fieldvalues"]
+ ;; ["databases" "my-db" "tables" "customers" "fields" "customer_id___fieldvalues"]
+ (fn [path]
+ (when-let [{db "databases"
+ schema "schemas"
+ table "tables"
+ field "fields"} (and (#{6 8} (count path))
+ (str/ends-with? (last path) field-values-slug)
+ (serdes.base/ingestion-matcher-pairs
+ path [["databases" "schemas" "tables" "fields"]
+ ["databases" "tables" "fields"]]))]
+ (filterv identity [{:model "Database" :id db}
+ (when schema {:model "Schema" :id schema})
+ {:model "Table" :id table}
+ {:model "Field" :id (subs field 0 (- (count field) (count field-values-slug)))}
+ ;; FieldValues is always just ID 0, since there's at most one as part of the field.
+ {:model "FieldValues" :id "0"}]))))
diff --git a/src/metabase/models/metric.clj b/src/metabase/models/metric.clj
index 8273bbf23acbe5c181d2fbe0d9039057ec910d1d..9ea0737a8d0e1442bb5a534f7d6320e37e420f8b 100644
--- a/src/metabase/models/metric.clj
+++ b/src/metabase/models/metric.clj
@@ -94,6 +94,16 @@
(into [] (set/union #{(serdes.util/table->path table_id)}
(serdes.util/mbql-deps definition))))
+(defmethod serdes.base/storage-path "Metric" [metric _ctx]
+ (let [{:keys [id label]} (-> metric serdes.base/serdes-path last)]
+ (-> metric
+ :table_id
+ serdes.util/table->path
+ serdes.util/storage-table-path-prefix
+ (concat ["metrics" (serdes.base/storage-leaf-file-name id label)]))))
+
+(serdes.base/register-ingestion-path! "Metric" (serdes.base/ingestion-matcher-collected "databases" "Metric"))
+
;;; ----------------------------------------------------- OTHER ------------------------------------------------------
(s/defn retrieve-metrics :- [(mi/InstanceOf Metric)]
diff --git a/src/metabase/models/native_query_snippet.clj b/src/metabase/models/native_query_snippet.clj
index b605e810ebb28e4eeebdbe12aaad59134e735177..a64e3a4dbd0d687957edc4aec24300428ec35726 100644
--- a/src/metabase/models/native_query_snippet.clj
+++ b/src/metabase/models/native_query_snippet.clj
@@ -75,10 +75,8 @@
;;; ------------------------------------------------- Serialization --------------------------------------------------
-(defmethod serdes.base/extract-query "NativeQuerySnippet" [_ {:keys [collection-set]}]
- (eduction cat [(db/select-reducible NativeQuerySnippet :collection_id nil)
- (when (seq collection-set)
- (db/select-reducible NativeQuerySnippet :collection_id [:in collection-set]))]))
+(defmethod serdes.base/extract-query "NativeQuerySnippet" [_ opts]
+ (serdes.base/extract-query-collections NativeQuerySnippet opts))
(defmethod serdes.base/extract-one "NativeQuerySnippet"
[_model-name _opts snippet]
@@ -97,3 +95,21 @@
(if collection_id
[[{:model "Collection" :id collection_id}]]
[]))
+
+(defmethod serdes.base/storage-path "NativeQuerySnippet" [snippet ctx]
+ ;; Intended path here is ["snippets" "nested" "collections" "snippet_eid_and_slug"]
+ ;; We just the default path, then pull it apart.
+ ;; The default is ["collections" "nested" collections" "nativequerysnippets" "base_name"]
+ (let [basis (serdes.base/storage-default-collection-path snippet ctx)
+ file (last basis)
+ colls (->> basis rest (drop-last 2))] ; Drops the "collections" at the start, and the last two.
+ (concat ["snippets"] colls [file])))
+
+(serdes.base/register-ingestion-path!
+ "NativeQuerySnippet"
+ (fn [path]
+ (when-let [[id slug] (and (= (first path) "snippets")
+ (serdes.base/split-leaf-file-name (last path)))]
+ (cond-> {:model "NativeQuerySnippet" :id id}
+ slug (assoc :label slug)
+ true vector))))
diff --git a/src/metabase/models/segment.clj b/src/metabase/models/segment.clj
index 0f446e9541c9089e67aa81c502fba85281b148e2..48aa9a316f84af28246475ce252060b8ebce9ed7 100644
--- a/src/metabase/models/segment.clj
+++ b/src/metabase/models/segment.clj
@@ -94,6 +94,16 @@
(into [] (set/union #{(serdes.util/table->path table_id)}
(serdes.util/mbql-deps definition))))
+(defmethod serdes.base/storage-path "Segment" [segment _ctx]
+ (let [{:keys [id label]} (-> segment serdes.base/serdes-path last)]
+ (-> segment
+ :table_id
+ serdes.util/table->path
+ serdes.util/storage-table-path-prefix
+ (concat ["segments" (serdes.base/storage-leaf-file-name id label)]))))
+
+(serdes.base/register-ingestion-path! "Segment" (serdes.base/ingestion-matcher-collected "databases" "Segment"))
+
;;; ------------------------------------------------------ Etc. ------------------------------------------------------
(s/defn retrieve-segments :- [(mi/InstanceOf Segment)]
diff --git a/src/metabase/models/serialization/base.clj b/src/metabase/models/serialization/base.clj
index d2fac6c0ab8c2264d571679a82d1633fd21ce9ef..74b3f7935f3611ecefa37cade3bd7ed823294f32 100644
--- a/src/metabase/models/serialization/base.clj
+++ b/src/metabase/models/serialization/base.clj
@@ -9,7 +9,8 @@
If the model is not exported, add it to the exclusion lists in the tests. Every model should be explicitly listed as
exported or not, and a test enforces this so serialization isn't forgotten for new models."
- (:require [clojure.tools.logging :as log]
+ (:require [clojure.string :as str]
+ [clojure.tools.logging :as log]
[metabase.models.interface :as mi]
[metabase.models.serialization.hash :as serdes.hash]
[metabase.util :as u]
@@ -152,6 +153,18 @@
;;; The storage system might transform that stream in some arbitrary way. Storage is a dead end - it should perform side
;;; effects like writing to the disk or network, and return nothing.
;;;
+;;; Not all storage solutions use directory structure, but for those that do, [[storage-path]] should give the path as
+;;; a list of strings: `["foo" "bar" "some_file"]`. Note the lack of a file extension on the last segment - that
+;;; is deliberately left off so that no filename surgery is required to support eg. both JSON and YAML output.
+;;;
+;;; By convention, models are named as plural and in lower case:
+;;; `["collections" "1234ABC_my_collection" "dashboards" "8765def_health_metrics"]`.
+;;;
+;;; As a final remark, note that some entities have their own directories and some do not. For example a Field is
+;;; simply a file, while a Table has a directory. So a Table's itself is
+;;; `["databases" "some-db" "schemas" "PUBLIC" "tables" "Customer" "Customer"]`
+;;; so that's a directory called `Customer` with a file called (for YAML output) `Customer.yaml` in it.
+;;;
;;; Selective Serialization:
;;; Sometimes we want to export a "subtree" instead of the complete appdb. At the simplest, we might serialize a single
;;; question. Moving up, it might be a Dashboard and all its questions, or a Collection and all its content Cards and
@@ -211,6 +224,19 @@
(eduction (map (partial extract-one model opts))
(extract-query model opts)))
+(defn extract-query-collections
+ "Helper for the common (but not default) [[extract-query]] case of fetching everything that isn't in a personal
+ collection."
+ [model {:keys [collection-set]}]
+ (if collection-set
+ ;; If collection-set is defined, select everything in those collections, or with nil :collection_id.
+ (let [in-colls (db/select-reducible model :collection_id [:in collection-set])]
+ (if (contains? collection-set nil)
+ (eduction cat [in-colls (db/select-reducible model :collection_id nil)])
+ in-colls))
+ ;; If collection-set is nil, just select everything.
+ (db/select-reducible model)))
+
(defmethod extract-query :default [model-name _]
(db/select-reducible (symbol model-name)))
@@ -472,3 +498,136 @@
(if (entity-id? id-str)
(db/select-one model :entity_id id-str)
(find-by-identity-hash model id-str)))
+
+;;; +----------------------------------------------------------------------------------------------------------------+
+;;; | Storage |
+;;; +----------------------------------------------------------------------------------------------------------------+
+;;; These storage multimethods take a second argument known as the context. This is a good place for a particular
+;;; storage implementation to include some precomputed information, or options.
+;;; In particular, it should include a table of collection IDs to path fragments that is precomputed by the host.
+;;; [[storage-base-context]] computes that, since many things go in a collections tree.
+(def ^:private max-label-length 100)
+
+(defn- truncate-label [s]
+ (if (> (count s) max-label-length)
+ (subs s 0 max-label-length)
+ s))
+
+(defn- lower-plural [s]
+ (-> s u/lower-case-en (str "s")))
+
+(defn storage-leaf-file-name
+ "Captures the common pattern for leaf file names as `entityID_label`."
+ ([id] (str id))
+ ([id label] (if (nil? label)
+ (storage-leaf-file-name id)
+ (str id "_" (truncate-label label)))))
+
+(defn storage-default-collection-path
+ "Implements the most common structure for [[storage-path]] - `collections/c1/c2/c3/models/entityid_slug.ext`"
+ [entity {:keys [collections]}]
+ (let [{:keys [model id label]} (-> entity serdes-path last)]
+ (concat ["collections"]
+ (get collections (:collection_id entity)) ;; This can be nil, but that's fine - that's the root collection.
+ [(lower-plural model) (storage-leaf-file-name id label)])))
+
+(defmulti storage-path
+ "Computes the complete storage path for a given entity.
+ `(storage-path entity ctx)`
+ Dispatches on the model name, eg. \"Dashboard\".
+
+ Returns a list of strings giving the path, with the final entry being the file name with no extension.
+
+ The default implementation works for entities which are:
+ - Part of the regular (not snippet) collections tree, per a :collection_id field; and
+ - Stored as `foos/1234abc_slug.extension` underneath their collection
+ eg. Cards, Dashboards, Timelines
+
+ The default logic is captured by [[storage-default-collection-path]] so it can be reused."
+ {:arglists '([entity ctx])}
+ (fn [entity _] (ingested-model entity)))
+
+(defmethod storage-path :default [entity ctx]
+ (storage-default-collection-path entity ctx))
+
+(defn storage-base-context
+ "Creates the basic context for storage. This is a map with a single entry: `:collections` is a map from collection ID
+ to the path of collections."
+ []
+ (let [colls (db/select ['Collection :id :entity_id :location :slug])
+ coll-names (into {} (for [{:keys [id entity_id slug]} colls]
+ [(str id) (storage-leaf-file-name entity_id slug)]))
+ coll->path (into {} (for [{:keys [entity_id id location]} colls
+ :let [parents (rest (str/split location #"/"))]]
+ [entity_id (map coll-names (concat parents [(str id)]))]))]
+ {:collections coll->path}))
+
+;;; +----------------------------------------------------------------------------------------------------------------+
+;;; | Ingestion |
+;;; +----------------------------------------------------------------------------------------------------------------+
+(defonce ^:private ingest-path-matchers (atom {}))
+
+(defn ingest-path
+ "Transforms a file path (as a sequence of strings with no file extension on the last part) into a `:serdes/meta`
+ hierarchy, or nil if nothing matches.
+
+ The set of matchers is maintained in an atom, and each model should call [[register-ingestion-path!]] to add its
+ matcher.
+
+ Note that the input format is the same as the return value of [[storage-path]]."
+ [path]
+ (first (keep #(% path) (vals @ingest-path-matchers))))
+
+(defn register-ingestion-path!
+ "Registers the matcher for the given model. Expects the model name to be a string.
+ The matcher is a function from a path (see [[ingest-path]] for the format) to a `:serdes/meta` hierarchy, or nil
+ if there's no match."
+ [model-name matcher]
+ (swap! ingest-path-matchers assoc model-name matcher)
+ ;; Return a readable symbol so this call shows up nicely when (a buffer containing) a register-ingestion-path! call
+ ;; is evaluated at the REPL.
+ ['register-ingestion-path! model-name])
+
+(defn split-leaf-file-name
+ "Given a leaf file name of the type generated by [[storage-leaf-file-name]], break it apart into an [id slug] pair,
+ where the slug might be nil."
+ [file]
+ (when-let [[_ id slug] (or (re-matches #"^([A-Za-z0-9_\.:-]{21})_(.*)$" file) ; entity_id and slug
+ (re-matches #"^([A-Za-z0-9_\.:-]{21})$" file) ; entity_id only
+ (re-matches #"^([a-fA-F0-9]{8})_(.*)$" file) ; Hash and slug
+ (re-matches #"^([a-fA-F0-9]{8})$" file))] ; Hash only
+ [id slug]))
+
+(defn ingestion-matcher-collected
+ "A helper for the common case of paths like `collections/some/nested/collections/model-name/entityID_slug`.
+ Expects the (lowercase) first segment, and the model name (eg. \"Dashboard\", not \"dashboards\" as it appears in
+ the path).
+ For example `(ingestion-matcher-collected \"collections\" \"Card\")`.
+
+ Returns a matcher function.
+ The resulting hierarchy is "
+ [first-segment model-name]
+ (fn [path]
+ (let [head (first path)
+ [model file] (take-last 2 path)]
+ (when-let [[id slug] (and (= head first-segment)
+ (= model (lower-plural model-name))
+ (split-leaf-file-name file))]
+ (cond-> {:model model-name :id id}
+ slug (assoc :label slug)
+ true vector)))))
+
+(defn- match-pairs [path pattern]
+ (let [chunks (take (count pattern) (partition 2 path))]
+ (when (= pattern (map first chunks))
+ (reduce (fn [out [k v]] (assoc out k v)) {} chunks))))
+
+(defn ingestion-matcher-pairs
+ "A helper for the common case of paths like `databases/my-db/schemas/my-schema/tables/my-table` which alternate
+ fixed and arbitrary segments.
+ The input is a *list* of sequences like `[[\"databases\" \"schemas\" \"tables\"] [\"databases\" \"tables\"]]` and the
+ response is a map of those fixed tokens as keys to the following segment as values, eg.
+ `{\"databases\" \"my-db\" \"schemas\" \"my-schema\" \"tables\" \"my-table\"}`.
+ This matches a *prefix*, not necessarily the entire sequence."
+ [path patterns]
+ (some (partial match-pairs path) patterns))
diff --git a/src/metabase/models/serialization/util.clj b/src/metabase/models/serialization/util.clj
index 2486ffab2a8b794033a860ad2504d7164f10cc2f..426eeeab7bb57d6994397e2afba3027b8cd69618 100644
--- a/src/metabase/models/serialization/util.clj
+++ b/src/metabase/models/serialization/util.clj
@@ -124,6 +124,23 @@
(when schema {:model "Schema" :id schema})
{:model "Table" :id table-name}]))
+(defn storage-table-path-prefix
+ "The [[serdes.base/storage-path]] for Table is a bit tricky, and shared with Fields and FieldValues, so it's
+ factored out here.
+ Takes the :serdes/meta value for a `Table`!
+ The return value includes the directory for the Table, but not the file for the Table itself.
+
+ With a schema: `[\"databases\" \"db_name\" \"schemas\" \"public\" \"tables\" \"customers\"]`
+ No schema: `[\"databases\" \"db_name\" \"tables\" \"customers\"]`"
+ [path]
+ (let [db-name (-> path first :id)
+ schema (when (= (count path) 3)
+ (-> path second :id))
+ table-name (-> path last :id)]
+ (concat ["databases" db-name]
+ (when schema ["schemas" schema])
+ ["tables" table-name])))
+
;; -------------------------------------------------- Fields ---------------------------------------------------------
(defn export-field-fk
"Given a numeric `field_id`, return a portable field reference.
@@ -328,6 +345,10 @@
["field" (field :guard vector?) tail] (into #{(field->path field)} (mbql-deps-map tail))
[:field-id (field :guard vector?) tail] (into #{(field->path field)} (mbql-deps-map tail))
["field-id" (field :guard vector?) tail] (into #{(field->path field)} (mbql-deps-map tail))
+ [:metric (field :guard portable-id?)] #{[{:model "Metric" :id field}]}
+ ["metric" (field :guard portable-id?)] #{[{:model "Metric" :id field}]}
+ [:segment (field :guard portable-id?)] #{[{:model "Segment" :id field}]}
+ ["segment" (field :guard portable-id?)] #{[{:model "Segment" :id field}]}
:else (reduce #(cond
(map? %2) (into %1 (mbql-deps-map %2))
(vector? %2) (into %1 (mbql-deps-vector %2))
diff --git a/src/metabase/models/table.clj b/src/metabase/models/table.clj
index f4dba6eb19e2520f766ca75efacf5d1549264640..874d43bd241dbc267479271d563abd978db632a3 100644
--- a/src/metabase/models/table.clj
+++ b/src/metabase/models/table.clj
@@ -13,6 +13,7 @@
[metabase.models.segment :refer [retrieve-segments Segment]]
[metabase.models.serialization.base :as serdes.base]
[metabase.models.serialization.hash :as serdes.hash]
+ [metabase.models.serialization.util :as serdes.util]
[metabase.util :as u]
[toucan.db :as db]
[toucan.models :as models]))
@@ -264,3 +265,23 @@
[{:keys [db_id] :as table}]
(-> (serdes.base/load-xform-basics table)
(assoc :db_id (db/select-one-field :id 'Database :name db_id))))
+
+(defmethod serdes.base/storage-path "Table" [table _ctx]
+ (concat (serdes.util/storage-table-path-prefix (serdes.base/serdes-path table))
+ [(:name table)]))
+
+(serdes.base/register-ingestion-path!
+ "Table"
+ ;; ["databases" "my-db" "schemas" "PUBLIC" "tables" "customers" "customers"]
+ ;; ["databases" "my-db" "tables" "customers" "customers"]
+ ;; Note that the last 2 must match, they're the table's directory and its file.
+ (fn [path]
+ (when-let [{db "databases"
+ schema "schemas"
+ table "tables"} (and (#{5 7} (count path))
+ (apply = (take-last 2 path))
+ (serdes.base/ingestion-matcher-pairs path [["databases" "schemas" "tables"]
+ ["databases" "tables"]]))]
+ (filterv identity [{:model "Database" :id db}
+ (when schema {:model "Schema" :id schema})
+ {:model "Table" :id table}]))))
diff --git a/src/metabase/models/timeline.clj b/src/metabase/models/timeline.clj
index 18b06aec80e5a7425d6268065be2820fbacfb44a..8ec5de88a76861374db48b924406ff35d178a52b 100644
--- a/src/metabase/models/timeline.clj
+++ b/src/metabase/models/timeline.clj
@@ -65,17 +65,15 @@
[:name (serdes.hash/hydrated-hash :collection "<none>") :created_at])
;;;; serialization
-(defmethod serdes.base/extract-query "Timeline" [_model-name {:keys [collection-set]}]
+(defmethod serdes.base/extract-query "Timeline" [_model-name opts]
(eduction (map #(timeline-event/include-events-singular % {:all? true}))
- (if (seq collection-set)
- (db/select-reducible Timeline :collection_id [:in collection-set])
- (db/select-reducible Timeline))))
+ (serdes.base/extract-query-collections Timeline opts)))
(defn- extract-events [events]
(sort-by :timestamp
(for [event events]
(-> (into (sorted-map) event)
- (dissoc :creator :id :timeline_id)
+ (dissoc :creator :id :timeline_id :updated_at)
(update :creator_id serdes.util/export-user)
(update :timestamp #(u.date/format (t/offset-date-time %)))))))
@@ -107,3 +105,5 @@
(defmethod serdes.base/serdes-dependencies "Timeline" [{:keys [collection_id]}]
[[{:model "Collection" :id collection_id}]])
+
+(serdes.base/register-ingestion-path! "Timeline" (serdes.base/ingestion-matcher-collected "collections" "Timeline"))
diff --git a/src/metabase/models/timeline_event.clj b/src/metabase/models/timeline_event.clj
index 81f258af1e462cb159e73d70da8db2c2a1285ce4..2d19e37a914ec73f855808a8b052f9fe7ba90cf9 100644
--- a/src/metabase/models/timeline_event.clj
+++ b/src/metabase/models/timeline_event.clj
@@ -110,4 +110,5 @@
serdes.base/load-xform-basics
(update :timeline_id serdes.util/import-fk 'Timeline)
(update :creator_id serdes.util/import-user)
- (update :timestamp u.date/parse)))
+ (update :timestamp u.date/parse)
+ (update :created_at #(if (string? %) (u.date/parse %) %))))