From f0655fc253ca2f17af32d342115a833753b1119e Mon Sep 17 00:00:00 2001
From: Braden Shepherdson <braden@metabase.com>
Date: Thu, 1 Dec 2022 11:11:01 -0500
Subject: [PATCH] Serdes v2: Rebuild the directory structure to be more
human-friendly (#26793)
There are now three top-level trees:
- regular `collections/path/to/collection/...`
- `:namespace :snippet` collections in `snippets/path/to/collection/...`
- `databases/mydb/schemas/PUBLIC/tables/customers/fields/name.yaml`
The path for any given entity is determined by the
`serdes.base/storage-path` multimethod.
On the ingestion side, things are a bit tricky because the paths don't
map directly to `:serdes/meta` hierarchies anymore. Instead each model
registers a function to turn a file path into either a `:serdes/meta`
hierarcy or nil for a bad match.
Ingestion will fetch all these functions once and then try them all in
arbitrary order until one matches.
---
.../serialization/v2/backfill_ids.clj | 2 +-
.../serialization/v2/extract.clj | 3 +-
.../serialization/v2/ingest/yaml.clj | 64 +++---
.../serialization/v2/models.clj | 11 +-
.../serialization/v2/storage/yaml.clj | 7 +-
.../serialization/v2/utils/yaml.clj | 67 ++----
.../serialization/api/serialize_test.clj | 26 ++-
.../serialization/v2/e2e/yaml_test.clj | 199 +++++++++---------
.../serialization/v2/ingest/yaml_test.clj | 61 +++---
.../serialization/v2/load_test.clj | 74 +------
.../serialization/v2/storage/yaml_test.clj | 120 ++++++++---
src/metabase/models/card.clj | 31 +--
src/metabase/models/collection.clj | 15 ++
src/metabase/models/dashboard.clj | 8 +-
src/metabase/models/dashboard_card.clj | 2 +
src/metabase/models/database.clj | 13 ++
src/metabase/models/field.clj | 24 +++
src/metabase/models/field_values.clj | 34 ++-
src/metabase/models/metric.clj | 10 +
src/metabase/models/native_query_snippet.clj | 24 ++-
src/metabase/models/segment.clj | 10 +
src/metabase/models/serialization/base.clj | 161 +++++++++++++-
src/metabase/models/serialization/util.clj | 21 ++
src/metabase/models/table.clj | 21 ++
src/metabase/models/timeline.clj | 10 +-
src/metabase/models/timeline_event.clj | 3 +-
26 files changed, 681 insertions(+), 340 deletions(-)
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/backfill_ids.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/backfill_ids.clj
index 60e59658e5e..0d398c11e63 100644
--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/backfill_ids.clj
+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/backfill_ids.clj
@@ -34,7 +34,7 @@
`entity_id` set. If the `entity_id` is NULL, it is set based on the [[serdes.hash/identity-hash]] for that
row."
[]
- (doseq [model-name serdes.models/exported-models
+ (doseq [model-name (concat serdes.models/exported-models serdes.models/inlined-models)
:let [model (db/resolve-model (symbol model-name))]
:when (has-entity-id? model)]
(backfill-ids-for model)))
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/extract.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/extract.clj
index 16735babf55..ceab8d48c1c 100644
--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/extract.clj
+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/extract.clj
@@ -30,7 +30,8 @@
set)]
(->> (concat unowned owned)
(map collection/descendant-ids)
- (reduce set/union top-ids))))
+ (reduce set/union top-ids)
+ (set/union #{nil}))))
(defn extract-metabase
"Extracts the appdb into a reducible stream of serializable maps, with `:serdes/meta` keys.
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/ingest/yaml.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/ingest/yaml.clj
index 39d42add793..4260eefef1f 100644
--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/ingest/yaml.clj
+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/ingest/yaml.clj
@@ -4,7 +4,6 @@
(:require [clojure.java.io :as io]
[medley.core :as m]
[metabase-enterprise.serialization.v2.ingest :as ingest]
- [metabase-enterprise.serialization.v2.models :as models]
[metabase-enterprise.serialization.v2.utils.yaml :as u.yaml]
[metabase.util.date-2 :as u.date]
[yaml.core :as yaml]
@@ -22,7 +21,6 @@
; We return a path of 1 item, the setting itself.
[{:model "Setting" :id (name k)}])))
-
(defn- build-metas [^File root-dir ^File file]
(let [path-parts (u.yaml/path-split root-dir file)]
(if (= ["settings.yaml"] path-parts)
@@ -46,6 +44,9 @@
(sequential? obj) (mapv keywords obj)
:else obj))
+(defn- strip-labels [hierarchy]
+ (mapv #(dissoc % :label) hierarchy))
+
(defn- ingest-entity
"Given a hierarchy, read in the YAML file it identifies. Clean it up (eg. parsing timestamps) and attach the
hierarchy as `:serdes/meta`.
@@ -54,42 +55,53 @@
The labels are removed from the hierarchy attached at `:serdes/meta`, since the storage system might have damaged the
original labels by eg. truncating them to keep the file names from getting too long. The labels aren't used at all on
the loading side, so it's fine to drop them."
- [root-dir hierarchy]
- (let [unlabeled (mapv #(dissoc % :label) hierarchy)
- file (u.yaml/hierarchy->file root-dir hierarchy)] ; Use the original hierarchy for the filesystem.
- (-> (when (.exists file) file) ; If the returned file doesn't actually exist, replace it with nil.
+ [hierarchy ^File file]
+ (-> (when (.exists file) file) ; If the returned file doesn't actually exist, replace it with nil.
+
+ ;; No automatic keywords; it's too generous with what counts as a keyword and has a bug.
+ ;; See https://github.com/clj-commons/clj-yaml/issues/64
+ (yaml/from-file false)
+ keywords
+ read-timestamps
+ (assoc :serdes/meta (strip-labels hierarchy)))) ; But return the hierarchy without labels.
+
+(def ^:private legal-top-level-paths
+ "These are all the legal first segments of paths. This is used by ingestion to avoid `.git`, `.github`, `README.md`
+ and other such extras."
+ #{"collections" "databases" "snippets" "settings.yaml"})
- ;; No automatic keywords; it's too generous with what counts as a keyword and has a bug.
- ;; See https://github.com/clj-commons/clj-yaml/issues/64
- (yaml/from-file false)
- keywords
- read-timestamps
- (assoc :serdes/meta unlabeled)))) ; But return the hierarchy without labels.
+(defn- ingest-all [^File root-dir]
+ ;; This returns a map {unlabeled-hierarchy [original-hierarchy File]}.
+ (into {} (for [^File file (file-seq root-dir)
+ :when (and (.isFile file)
+ (let [rel (.relativize (.toPath root-dir) (.toPath file))]
+ (-> rel (.subpath 0 1) (.toString) legal-top-level-paths)))
+ hierarchy (build-metas root-dir file)]
+ [(strip-labels hierarchy) [hierarchy file]])))
-(deftype YamlIngestion [^File root-dir settings]
+(deftype YamlIngestion [^File root-dir settings cache]
ingest/Ingestable
(ingest-list [_]
- (let [model-set (set models/exported-models)]
- (eduction (comp (filter (fn [^File f] (.isFile f)))
- ;; The immediate parent directory should be a recognized model name.
- ;; If it's not, this may be in .git, or .github/actions/... or similar extra files.
- (filter (fn [^File f] (or (= (.getName f) "settings.yaml")
- (-> f
- (.getParentFile)
- (.getName)
- model-set))))
- (mapcat (partial build-metas root-dir)))
- (file-seq root-dir))))
+ (->> (or @cache
+ (reset! cache (ingest-all root-dir)))
+ vals
+ (map first)))
(ingest-one [_ abs-path]
+ (when-not @cache
+ (reset! cache (ingest-all root-dir)))
(let [{:keys [model id]} (first abs-path)]
(if (and (= (count abs-path) 1)
(= model "Setting"))
{:serdes/meta abs-path :key (keyword id) :value (get settings (keyword id))}
- (ingest-entity root-dir abs-path)))))
+ (->> abs-path
+ strip-labels
+ (get @cache)
+ second
+ (ingest-entity abs-path))))))
(defn ingest-yaml
"Creates a new Ingestable on a directory of YAML files, as created by
[[metabase-enterprise.serialization.v2.storage.yaml]]."
[root-dir]
- (->YamlIngestion (io/file root-dir) (yaml/from-file (io/file root-dir "settings.yaml"))))
+ (->YamlIngestion (io/file root-dir) (yaml/from-file (io/file root-dir "settings.yaml")) (atom nil)))
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj
index f4d7c94b711..c5caad31a02 100644
--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj
+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj
@@ -10,10 +10,15 @@
"FieldValues"
"Metric"
"NativeQuerySnippet"
- "Pulse"
- "PulseCard"
- "PulseChannel"
"Segment"
"Setting"
"Table"
"Timeline"])
+
+(def inlined-models
+ "An additional list of models which are inlined into parent entities for serialization.
+ These are not extracted and serialized separately, but they may need some processing done.
+ For example, the models should also have their entity_id fields populated (if they have one)."
+ ["DashboardCard"
+ "Dimension"
+ "TimelineEvent"])
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/storage/yaml.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/storage/yaml.clj
index cb4aa742338..27214a6197d 100644
--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/storage/yaml.clj
+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/storage/yaml.clj
@@ -22,8 +22,8 @@
(io/make-parents file)
(spit (io/file file) (generate-yaml obj)))
-(defn- store-entity! [{:keys [root-dir]} entity]
- (spit-yaml (u.yaml/hierarchy->file root-dir (serdes.base/serdes-path entity))
+(defn- store-entity! [opts entity]
+ (spit-yaml (u.yaml/hierarchy->file opts entity)
(dissoc entity :serdes/meta)))
(defn- store-settings! [{:keys [root-dir]} settings]
@@ -37,7 +37,8 @@
(instance? java.io.File (:root-dir opts)))
(throw (ex-info ":yaml storage requires the :root-dir option to be a string or File"
{:opts opts})))
- (let [settings (atom [])]
+ (let [settings (atom [])
+ opts (merge opts (serdes.base/storage-base-context))]
(doseq [entity stream]
(if (-> entity :serdes/meta last :model (= "Setting"))
(swap! settings conj entity)
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/utils/yaml.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/utils/yaml.clj
index f40b6dc795e..6b89fb1e22e 100644
--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/utils/yaml.clj
+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/utils/yaml.clj
@@ -1,24 +1,10 @@
(ns metabase-enterprise.serialization.v2.utils.yaml
(:require [clojure.java.io :as io]
- [clojure.string :as str])
+ [clojure.string :as str]
+ [metabase.models.serialization.base :as serdes.base])
(:import java.io.File
java.nio.file.Path))
-(def ^:private max-label-length 100)
-
-(defn- truncate-label [s]
- (if (> (count s) max-label-length)
- (subs s 0 max-label-length)
- s))
-
-(defn- leaf-file-name
- ([id] (str id ".yaml"))
- ;; + is a legal, unescaped character on all common filesystems,
- ;; but doesn't appear in `identity-hash` or NanoID!
- ([id label] (if (nil? label)
- (leaf-file-name id)
- (str id "+" (truncate-label label) ".yaml"))))
-
(defn- escape-segment
"Given a path segment, which is supposed to be the name of a single file or directory, escape any slashes inside it.
This occurs in practice, for example with a `Field.name` containing a slash like \"Company/organization website\"."
@@ -35,26 +21,14 @@
(str/replace "__BACKSLASH__" "\\")))
(defn hierarchy->file
- "Given a :serdes/meta abstract path, return a [[File]] corresponding to it."
- ^File [root-dir hierarchy]
- (let [;; All earlier parts of the hierarchy form Model/id/ pairs.
- prefix (apply concat (for [{:keys [model id]} (drop-last hierarchy)]
- [model id]))
- ;; The last part of the hierarchy is used for the basename; this is the only part with the label.
- {:keys [id model label]} (last hierarchy)
- leaf-name (leaf-file-name id label)
- as-given (apply io/file root-dir (map escape-segment (concat prefix [model leaf-name])))]
- (if (.exists ^File as-given)
- as-given
- ; If that file name doesn't exist, check the directory to see if there's one that's the requested file plus a
- ; human-readable portion.
- (let [dir (apply io/file root-dir (map escape-segment (concat prefix [model])))
- matches (filter #(and (.startsWith ^String % (str id "+"))
- (.endsWith ^String % ".yaml"))
- (.list ^File dir))]
- (if (empty? matches)
- (io/file dir (escape-segment leaf-name))
- (io/file dir (first matches)))))))
+ "Given an extracted entity, return a [[File]] corresponding to it."
+ ^File [ctx entity]
+ (let [;; Get the desired [[serdes.base/storage-path]].
+ base-path (serdes.base/storage-path entity ctx)
+ dirnames (drop-last base-path)
+ ;; Attach the file extension to the last part.
+ basename (str (last base-path) ".yaml")]
+ (apply io/file (:root-dir ctx) (map escape-segment (concat dirnames [basename])))))
(defn path-split
"Given a root directory and a file underneath it, return a sequence of path parts to get there.
@@ -67,13 +41,16 @@
(defn path->hierarchy
"Given the list of file path chunks as returned by [[path-split]], reconstruct the `:serdes/meta` abstract path
corresponding to it.
- Note that the __SLASH__ and __BACKSLASH__ interpolations of [[escape-segment]] are reversed here."
+ Note that the __SLASH__ and __BACKSLASH__ interpolations of [[escape-segment]] are reversed here, and also the
+ file extension is stripped off the last segment.
+
+ The heavy lifting is done by the matcher functions registered by each model using
+ [[serdes.base/register-ingestion-path!]]."
[path-parts]
- (let [parentage (into [] (for [[model id] (partition 2 (drop-last 2 path-parts))]
- {:model model :id (unescape-segment id)}))
- [model basename] (take-last 2 path-parts)
- basename (unescape-segment basename)
- [_ id label] (or (re-matches #"^([A-Za-z0-9_\.:-]+)(?:\+(.*))?\.yaml$" basename)
- (re-matches #"^(.+)\.yaml$" basename))]
- (conj parentage (cond-> {:model model :id id}
- label (assoc :label label)))))
+ (let [basename (last path-parts)
+ basename (if (str/ends-with? basename ".yaml")
+ (subs basename 0 (- (count basename) 5))
+ basename)
+ path-parts (concat (map unescape-segment (drop-last path-parts))
+ [(unescape-segment basename)])]
+ (serdes.base/ingest-path path-parts)))
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/api/serialize_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/api/serialize_test.clj
index 17b771b2a23..b45ee49e713 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/api/serialize_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/api/serialize_test.clj
@@ -10,7 +10,9 @@
(defn- do-serialize-data-model [f]
(premium-features-test/with-premium-features #{:serialization}
- (mt/with-temp* [Collection [{collection-id :id}]
+ (mt/with-temp* [Collection [{collection-id :id
+ collection-eid :entity_id
+ collection-slug :slug}]
Dashboard [{dashboard-id :id} {:collection_id collection-id}]
Card [{card-id :id} {:collection_id collection-id}]
DashboardCard [_ {:card_id card-id, :dashboard_id dashboard-id}]]
@@ -19,11 +21,15 @@
(is (= collection-id
(db/select-one-field :collection_id Card :id card-id))))
(mt/with-temp-dir [dir "serdes-dir"]
- (f {:collection-id collection-id, :dir dir})))))
+ (f {:collection-id collection-id
+ :collection-filename (if collection-slug
+ (str collection-eid "_" collection-slug)
+ collection-eid)
+ :dir dir})))))
(deftest serialize-data-model-happy-path-test
(do-serialize-data-model
- (fn [{:keys [collection-id dir]}]
+ (fn [{:keys [collection-id collection-filename dir]}]
(is (= {:status "ok"}
(mt/user-http-request :crowberto :post 200 "ee/serialization/serialize/data-model"
{:collection_ids [collection-id]
@@ -35,18 +41,18 @@
(path-files (apply u.files/get-path dir path-components)))]
(is (= (map
#(.toString (u.files/get-path (System/getProperty "java.io.tmpdir") "serdes-dir" %))
- ["Card" "Collection" "Dashboard" "settings.yaml"])
+ ["collections" "settings.yaml"])
(files)))
(testing "subdirs"
- (testing "Card"
+ (testing "cards"
(is (= 1
- (count (files "Card")))))
- (testing "Collection"
+ (count (files "collections" collection-filename "cards")))))
+ (testing "collections"
(is (= 1
- (count (files "Collection")))))
- (testing "Dashboard"
+ (count (remove #{"cards" "dashboards" "timelines"} (files "collections"))))))
+ (testing "dashboards"
(is (= 1
- (count (files "Dashboard")))))))))))
+ (count (files "collections" collection-filename "dashboards")))))))))))
(deftest serialize-data-model-validation-test
(do-serialize-data-model
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e/yaml_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e/yaml_test.clj
index 5df75c314bb..5decc7568ad 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e/yaml_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e/yaml_test.clj
@@ -13,18 +13,39 @@
[toucan.db :as db]
[yaml.core :as yaml]))
-(defn- dir->file-set [dir]
+(defn- dir->contents-set [p dir]
(->> dir
.listFiles
- (filter #(.isFile %))
+ (filter p)
(map #(.getName %))
set))
+(defn- dir->file-set [dir]
+ (dir->contents-set #(.isFile %) dir))
+
+(defn- dir->dir-set [dir]
+ (dir->contents-set #(.isDirectory %) dir))
+
(defn- subdirs [dir]
(->> dir
.listFiles
(remove #(.isFile %))))
+(defn- collections [dir]
+ (for [coll-dir (subdirs dir)
+ :when (->> ["cards" "dashboards" "timelines"]
+ (map #(io/file coll-dir %))
+ (filter #(= % coll-dir))
+ empty?)]
+ coll-dir))
+
+(defn- file-set [dir]
+ (let [base (.toPath dir)]
+ (set (for [file (file-seq dir)
+ :when (.isFile file)
+ :let [rel (.relativize base (.toPath file))]]
+ (mapv str rel)))))
+
(defn- random-keyword
([prefix n] (random-keyword prefix n 0))
([prefix n floor] (keyword (str (name prefix) (+ floor (rand-int n))))))
@@ -63,11 +84,12 @@
(let [extraction (atom nil)
entities (atom nil)]
(ts/with-source-and-dest-dbs
- ;; TODO Generating some nested collections would make these tests more robust.
+ ;; TODO Generating some nested collections would make these tests more robust, but that's difficult.
+ ;; There are handwritten tests for storage and ingestion that check out the nesting, at least.
(ts/with-source-db
(testing "insert"
(test-gen/insert!
- {:collection [[100 {:refs {:personal_owner_id ::rs/omit}}]
+ {:collection [[100 {:refs {:personal_owner_id ::rs/omit}}]
[10 {:refs {:personal_owner_id ::rs/omit}
:spec-gen {:namespace :snippets}}]]
:database [[10]]
@@ -145,73 +167,88 @@
(update m (-> entity :serdes/meta last :model)
(fnil conj []) entity))
{} @extraction))
- (is (= 110 (-> @entities (get "Collection") count)))))
-
- (testing "storage"
- (storage.yaml/store! (seq @extraction) dump-dir)
-
- (testing "for Collections"
- (is (= 110 (count (dir->file-set (io/file dump-dir "Collection"))))))
-
- (testing "for Databases"
- (is (= 10 (count (dir->file-set (io/file dump-dir "Database"))))))
-
- (testing "for Tables"
- (is (= 100
- (reduce + (for [db (get @entities "Database")
- :let [tables (dir->file-set (io/file dump-dir "Database" (:name db) "Table"))]]
- (count tables))))
- "Tables are scattered, so the directories are harder to count"))
-
- (testing "for Fields"
- (is (= 1000
- (reduce + (for [db (get @entities "Database")
- table (subdirs (io/file dump-dir "Database" (:name db) "Table"))]
- (->> (io/file table "Field")
- dir->file-set
- count))))
- "Fields are scattered, so the directories are harder to count"))
-
- (testing "for cards"
- (is (= 100 (count (dir->file-set (io/file dump-dir "Card"))))))
-
- (testing "for dashboards"
- (is (= 100 (count (dir->file-set (io/file dump-dir "Dashboard"))))))
-
- (testing "for metrics"
- (is (= 30 (count (dir->file-set (io/file dump-dir "Metric"))))))
-
- (testing "for segments"
- (is (= 30 (count (dir->file-set (io/file dump-dir "Segment"))))))
-
- (testing "for pulses"
- (is (= 30 (count (dir->file-set (io/file dump-dir "Pulse"))))))
-
- (testing "for pulse cards"
- (is (= 120 (reduce + (for [pulse (get @entities "Pulse")]
- (->> (io/file dump-dir "Pulse" (:entity_id pulse) "PulseCard")
- dir->file-set
- count))))))
-
- (testing "for pulse channels"
- (is (= 30 (reduce + (for [pulse (get @entities "Pulse")]
- (->> (io/file dump-dir "Pulse" (:entity_id pulse) "PulseChannel")
- dir->file-set
- count)))))
- (is (= 40 (reduce + (for [{:keys [recipients]} (get @entities "PulseChannel")]
- (count recipients))))))
-
- (testing "for native query snippets"
- (is (= 10 (count (dir->file-set (io/file dump-dir "NativeQuerySnippet"))))))
-
- (testing "for timelines and events"
- (is (= 10 (count (dir->file-set (io/file dump-dir "Timeline"))))))
-
- (testing "for settings"
- (is (.exists (io/file dump-dir "settings.yaml")))))
+ (is (= 110 (-> @entities (get "Collection") count))))
+
+ (testing "storage"
+ (storage.yaml/store! (seq @extraction) dump-dir)
+
+ (testing "for Collections"
+ (is (= 110 (count (for [f (file-set (io/file dump-dir))
+ :when (and (= (first f) "collections")
+ (let [[a b] (take-last 2 f)]
+ (= b (str a ".yaml"))))]
+ f)))
+ "which all go in collections/, even the snippets ones"))
+
+ (testing "for Databases"
+ (is (= 10 (count (dir->dir-set (io/file dump-dir "databases"))))))
+
+ (testing "for Tables"
+ (is (= 100
+ (reduce + (for [db (get @entities "Database")
+ :let [tables (dir->dir-set (io/file dump-dir "databases" (:name db) "tables"))]]
+ (count tables))))
+ "Tables are scattered, so the directories are harder to count"))
+
+ (testing "for Fields"
+ (is (= 1000
+ (reduce + (for [db (get @entities "Database")
+ table (subdirs (io/file dump-dir "databases" (:name db) "tables"))]
+ (->> (io/file table "fields")
+ dir->file-set
+ count))))
+ "Fields are scattered, so the directories are harder to count"))
+
+ (testing "for cards"
+ (is (= 100 (->> (io/file dump-dir "collections")
+ collections
+ (map (comp count dir->file-set #(io/file % "cards")))
+ (reduce +)))))
+
+ (testing "for dashboards"
+ (is (= 100 (->> (io/file dump-dir "collections")
+ collections
+ (map (comp count dir->file-set #(io/file % "dashboards")))
+ (reduce +)))))
+
+ (testing "for timelines"
+ (is (= 10 (->> (io/file dump-dir "collections")
+ collections
+ (map (comp count dir->file-set #(io/file % "timelines")))
+ (reduce +)))))
+
+ (testing "for metrics"
+ (is (= 30 (reduce + (for [db (dir->dir-set (io/file dump-dir "databases"))
+ table (dir->dir-set (io/file dump-dir "databases" db "tables"))
+ :let [metrics-dir (io/file dump-dir "databases" db "tables" table "metrics")]
+ :when (.exists metrics-dir)]
+ (count (dir->file-set metrics-dir)))))))
+
+ (testing "for segments"
+ (is (= 30 (reduce + (for [db (dir->dir-set (io/file dump-dir "databases"))
+ table (dir->dir-set (io/file dump-dir "databases" db "tables"))
+ :let [segments-dir (io/file dump-dir "databases" db "tables" table "segments")]
+ :when (.exists segments-dir)]
+ (count (dir->file-set segments-dir)))))))
+
+ (testing "for native query snippets"
+ (is (= 10 (->> (io/file dump-dir "snippets")
+ collections
+ (map (comp count dir->file-set))
+ (reduce +)))))
+
+ (testing "for settings"
+ (is (.exists (io/file dump-dir "settings.yaml"))))))
(testing "ingest and load"
(ts/with-dest-db
+ (testing "ingested set matches extracted set"
+ (let [extracted-set (set (map (comp #'ingest.yaml/strip-labels serdes.base/serdes-path) @extraction))]
+ (is (= (count extracted-set)
+ (count @extraction)))
+ (is (= extracted-set
+ (set (keys (#'ingest.yaml/ingest-all (io/file dump-dir))))))))
+
(testing "doing ingestion"
(is (serdes.load/load-metabase (ingest.yaml/ingest-yaml dump-dir))
"successful"))
@@ -290,32 +327,6 @@
(serdes.base/extract-one "Segment" {})
clean-entity)))))
- (testing "for pulses"
- (doseq [{:keys [entity_id] :as pulse} (get @entities "Pulse")]
- (is (= (clean-entity pulse)
- (->> (db/select-one 'Pulse :entity_id entity_id)
- (serdes.base/extract-one "Pulse" {})
- clean-entity)))))
-
- (testing "for pulse cards"
- (doseq [{:keys [entity_id] :as card} (get @entities "PulseCard")]
- (is (= (clean-entity card)
- (->> (db/select-one 'PulseCard :entity_id entity_id)
- (serdes.base/extract-one "PulseCard" {})
- clean-entity)))))
-
- (testing "for pulse channels"
- (doseq [{:keys [entity_id] :as channel} (get @entities "PulseChannel")]
- ;; The :recipients list is in arbitrary order - turn them into sets for comparison.
- (is (= (-> channel
- (update :recipients set)
- clean-entity)
- (let [loaded-channel (->> (db/select-one 'PulseChannel :entity_id entity_id)
- (serdes.base/extract-one "PulseChannel" {}))]
- (-> loaded-channel
- (update :recipients set)
- clean-entity))))))
-
(testing "for native query snippets"
(doseq [{:keys [entity_id] :as snippet} (get @entities "NativeQuerySnippet")]
(is (= (clean-entity snippet)
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/ingest/yaml_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/ingest/yaml_test.clj
index e72c04a76ff..02f0e40c7a4 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/ingest/yaml_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/ingest/yaml_test.clj
@@ -8,28 +8,32 @@
(deftest basic-ingest-test
(ts/with-random-dump-dir [dump-dir "serdesv2-"]
- (io/make-parents dump-dir "Collection" "fake") ; Prepare the right directories.
+ (io/make-parents dump-dir "collections" "1234567890abcdefABCDE_the_label" "fake") ; Prepare the right directories.
+ (io/make-parents dump-dir "collections" "0987654321zyxwvuABCDE" "fake")
+
(spit (io/file dump-dir "settings.yaml")
(yaml/generate-string {:some-key "with string value"
:another-key 7
:blank-key nil}))
- (spit (io/file dump-dir "Collection" "fake-id+the_label.yaml")
- (yaml/generate-string {:some "made up" :data "here" :entity_id "fake-id" :slug "the_label"}))
- (spit (io/file dump-dir "Collection" "no-label.yaml")
- (yaml/generate-string {:some "other" :data "in this one" :entity_id "no-label"}))
+ (spit (io/file dump-dir "collections" "1234567890abcdefABCDE_the_label" "1234567890abcdefABCDE_the_label.yaml")
+ (yaml/generate-string {:some "made up" :data "here" :entity_id "1234567890abcdefABCDE" :slug "the_label"}))
+ (spit (io/file dump-dir "collections" "0987654321zyxwvuABCDE" "0987654321zyxwvuABCDE.yaml")
+ (yaml/generate-string {:some "other" :data "in this one" :entity_id "0987654321zyxwvuABCDE"}))
(let [ingestable (ingest.yaml/ingest-yaml dump-dir)
- exp-files {[{:model "Collection" :id "fake-id" :label "the_label"}] {:some "made up"
- :data "here"
- :entity_id "fake-id"
- :slug "the_label"}
- [{:model "Collection" :id "no-label"}] {:some "other"
- :data "in this one"
- :entity_id "no-label"}
- [{:model "Setting" :id "some-key"}] {:key :some-key :value "with string value"}
- [{:model "Setting" :id "another-key"}] {:key :another-key :value 7}
- [{:model "Setting" :id "blank-key"}] {:key :blank-key :value nil}}]
- (testing "the right set of file is returned by ingest-list"
+ exp-files {[{:model "Collection"
+ :id "1234567890abcdefABCDE"
+ :label "the_label"}] {:some "made up"
+ :data "here"
+ :entity_id "1234567890abcdefABCDE"
+ :slug "the_label"}
+ [{:model "Collection" :id "0987654321zyxwvuABCDE"}] {:some "other"
+ :data "in this one"
+ :entity_id "0987654321zyxwvuABCDE"}
+ [{:model "Setting" :id "some-key"}] {:key :some-key :value "with string value"}
+ [{:model "Setting" :id "another-key"}] {:key :another-key :value 7}
+ [{:model "Setting" :id "blank-key"}] {:key :blank-key :value nil}}]
+ (testing "the right set of files is returned by ingest-list"
(is (= (set (keys exp-files))
(into #{} (ingest/ingest-list ingestable)))))
@@ -46,29 +50,30 @@
(deftest flexible-file-matching-test
(ts/with-random-dump-dir [dump-dir "serdesv2-"]
- (io/make-parents dump-dir "Collection" "fake")
- (spit (io/file dump-dir "Collection" "entity-id+human-readable-things.yaml")
+ (io/make-parents dump-dir "collections" "1234567890abcdefABCDE_human-readable-things" "fake")
+ (spit (io/file dump-dir "collections" "1234567890abcdefABCDE_human-readable-things"
+ "1234567890abcdefABCDE_human-readable-things.yaml")
(yaml/generate-string {:some "made up" :data "here"}))
(let [ingestable (ingest.yaml/ingest-yaml dump-dir)
exp {:some "made up"
:data "here"
- :serdes/meta [{:model "Collection" :id "entity-id"}]}]
+ :serdes/meta [{:model "Collection" :id "1234567890abcdefABCDE"}]}]
(testing "the returned set of files has the human-readable labels"
- (is (= #{[{:model "Collection" :id "entity-id" :label "human-readable-things"}]}
+ (is (= #{[{:model "Collection" :id "1234567890abcdefABCDE" :label "human-readable-things"}]}
(into #{} (ingest/ingest-list ingestable)))))
(testing "fetching the file with the label works"
(is (= exp
- (ingest/ingest-one ingestable [{:model "Collection" :id "entity-id" :label "human-readable-things"}]))))
+ (ingest/ingest-one ingestable [{:model "Collection" :id "1234567890abcdefABCDE" :label "human-readable-things"}]))))
(testing "fetching the file without the label also works"
(is (= exp
- (ingest/ingest-one ingestable [{:model "Collection" :id "entity-id"}])))))))
+ (ingest/ingest-one ingestable [{:model "Collection" :id "1234567890abcdefABCDE"}])))))))
(deftest file-name-escaping-test
(ts/with-random-dump-dir [dump-dir "serdesv2-"]
- (io/make-parents dump-dir "Database" "fake")
- (spit (io/file dump-dir "Database" "my data__SLASH__speculations.yaml")
+ (io/make-parents dump-dir "databases" "my data__SLASH__speculations" "fake")
+ (spit (io/file dump-dir "databases" "my data__SLASH__speculations" "my data__SLASH__speculations.yaml")
(yaml/generate-string {:some "made up" :data "here"}))
(let [ingestable (ingest.yaml/ingest-yaml dump-dir)
@@ -85,14 +90,14 @@
(deftest keyword-reconstruction-test
(ts/with-random-dump-dir [dump-dir "serdesv2-"]
- (io/make-parents dump-dir "Card" "fake")
- (spit (io/file dump-dir "Card" "some-card.yaml")
+ (io/make-parents dump-dir "collections" "cards" "fake")
+ (spit (io/file dump-dir "collections" "cards" "1234567890abcdefABCDE_some_card.yaml")
(yaml/generate-string {:visualization_settings
{:column_settings {"[\"name\",\"sum\"]" {:number_style "currency"}}}}))
(let [ingestable (ingest.yaml/ingest-yaml dump-dir)
exp {:visualization_settings {:column_settings {"[\"name\",\"sum\"]" {:number_style "currency"}}}
- :serdes/meta [{:model "Card" :id "some-card"}]}]
+ :serdes/meta [{:model "Card" :id "1234567890abcdefABCDE"}]}]
(testing "the file as read in correctly reconstructs keywords only where legal"
(is (= exp
- (ingest/ingest-one ingestable [{:model "Card" :id "some-card"}])))))))
+ (ingest/ingest-one ingestable [{:model "Card" :id "1234567890abcdefABCDE"}])))))))
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj
index 78453c21356..340c631ef35 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj
@@ -5,8 +5,8 @@
[metabase-enterprise.serialization.v2.extract :as serdes.extract]
[metabase-enterprise.serialization.v2.ingest :as serdes.ingest]
[metabase-enterprise.serialization.v2.load :as serdes.load]
- [metabase.models :refer [Card Collection Dashboard DashboardCard Database Field FieldValues Metric Pulse
- PulseChannel PulseChannelRecipient Segment Table Timeline TimelineEvent User]]
+ [metabase.models :refer [Card Collection Dashboard DashboardCard Database Field FieldValues Metric
+ Segment Table Timeline TimelineEvent User]]
[metabase.models.serialization.base :as serdes.base]
[schema.core :as s]
[toucan.db :as db])
@@ -163,76 +163,6 @@
(is (db/exists? Table :name "posts" :db_id (:id @db1d)))
(is (db/exists? Table :name "posts" :db_id (:id @db2d)))))))))
-(deftest pulse-channel-recipient-merging-test
- (testing "pulse channel recipients are listed as emails on a channel, then merged with the existing ones"
- (let [serialized (atom nil)
- u1s (atom nil)
- u2s (atom nil)
- u3s (atom nil)
- pulse-s (atom nil)
- pc1s (atom nil)
- pc2s (atom nil)
- pcr1s (atom nil)
- pcr2s (atom nil)
-
- u1d (atom nil)
- u2d (atom nil)
- u3d (atom nil)
- pulse-d (atom nil)
- pc1d (atom nil)]
- (ts/with-source-and-dest-dbs
- (testing "serializing the pulse, channel and recipients"
- (ts/with-source-db
- (reset! u1s (ts/create! User :first_name "Alex" :last_name "Lifeson" :email "alifeson@rush.yyz"))
- (reset! u2s (ts/create! User :first_name "Geddy" :last_name "Lee" :email "glee@rush.yyz"))
- (reset! u3s (ts/create! User :first_name "Neil" :last_name "Peart" :email "neil@rush.yyz"))
- (reset! pulse-s (ts/create! Pulse :name "Heartbeat" :creator_id (:id @u1s)))
- (reset! pc1s (ts/create! PulseChannel
- :pulse_id (:id @pulse-s)
- :channel_type :email
- :schedule_type :daily
- :schedule_hour 16))
- (reset! pc2s (ts/create! PulseChannel
- :pulse_id (:id @pulse-s)
- :channel_type :slack
- :schedule_type :hourly))
- ;; Only Lifeson and Lee are recipients in the source.
- (reset! pcr1s (ts/create! PulseChannelRecipient :pulse_channel_id (:id @pc1s) :user_id (:id @u1s)))
- (reset! pcr2s (ts/create! PulseChannelRecipient :pulse_channel_id (:id @pc1s) :user_id (:id @u2s)))
- (reset! serialized (into [] (serdes.extract/extract-metabase {})))))
-
- (testing "recipients are serialized as :recipients [email] on the PulseChannel"
- (is (= #{["alifeson@rush.yyz" "glee@rush.yyz"]
- []}
- (set (map :recipients (by-model @serialized "PulseChannel"))))))
-
- (testing "deserialization merges the existing recipients with the new ones"
- (ts/with-dest-db
- ;; Users in a different order, so different IDs.
- (reset! u2d (ts/create! User :first_name "Geddy" :last_name "Lee" :email "glee@rush.yyz"))
- (reset! u1d (ts/create! User :first_name "Alex" :last_name "Lifeson" :email "alifeson@rush.yyz"))
- (reset! u3d (ts/create! User :first_name "Neil" :last_name "Peart" :email "neil@rush.yyz"))
- (reset! pulse-d (ts/create! Pulse :name "Heartbeat" :creator_id (:id @u1d) :entity_id (:entity_id @pulse-s)))
- (reset! pc1d (ts/create! PulseChannel
- :entity_id (:entity_id @pc1s)
- :pulse_id (:id @pulse-d)
- :channel_type :email
- :schedule_type :daily
- :schedule_hour 16))
- ;; Only Lee and Peart are recipients in the source.
- (ts/create! PulseChannelRecipient :pulse_channel_id (:id @pc1d) :user_id (:id @u2d))
- (ts/create! PulseChannelRecipient :pulse_channel_id (:id @pc1d) :user_id (:id @u3d))
-
- (is (= 2 (db/count PulseChannelRecipient)))
- (is (= #{(:id @u2d) (:id @u3d)}
- (db/select-field :user_id PulseChannelRecipient)))
-
- (serdes.load/load-metabase (ingestion-in-memory @serialized))
-
- (is (= 3 (db/count PulseChannelRecipient)))
- (is (= #{(:id @u1d) (:id @u2d) (:id @u3d)}
- (db/select-field :user_id PulseChannelRecipient)))))))))
-
(deftest card-dataset-query-test
;; Card.dataset_query is a JSON-encoded MBQL query, which contain database, table, and field IDs - these need to be
;; converted to a portable form and read back in.
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/storage/yaml_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/storage/yaml_test.clj
index 8542a969b36..9b6b3560a7a 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/storage/yaml_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/storage/yaml_test.clj
@@ -6,18 +6,18 @@
[metabase-enterprise.serialization.test-util :as ts]
[metabase-enterprise.serialization.v2.extract :as extract]
[metabase-enterprise.serialization.v2.storage.yaml :as storage.yaml]
- [metabase.models :refer [Collection Database Field FieldValues Table]]
+ [metabase.models :refer [Card Collection Dashboard Database Field FieldValues NativeQuerySnippet Table]]
[metabase.models.serialization.base :as serdes.base]
[metabase.util.date-2 :as u.date]
[toucan.db :as db]
[yaml.core :as yaml]))
-(defn- dir->file-set [dir]
- (->> dir
- .listFiles
- (filter #(.isFile %))
- (map #(.getName %))
- set))
+(defn- file-set [dir]
+ (let [base (.toPath dir)]
+ (set (for [file (file-seq dir)
+ :when (.isFile file)
+ :let [rel (.relativize base (.toPath file))]]
+ (mapv str rel)))))
(deftest basic-dump-test
(ts/with-random-dump-dir [dump-dir "serdesv2-"]
@@ -25,15 +25,16 @@
(ts/with-temp-dpc [Collection [parent {:name "Some Collection"}]
Collection [child {:name "Child Collection" :location (format "/%d/" (:id parent))}]]
(let [export (into [] (extract/extract-metabase nil))
- parent-filename (format "%s+some_collection.yaml" (:entity_id parent))
- child-filename (format "%s+child_collection.yaml" (:entity_id child))]
+ parent-filename (format "%s_some_collection" (:entity_id parent))
+ child-filename (format "%s_child_collection" (:entity_id child))]
(storage.yaml/store! export dump-dir)
(testing "the right files in the right places"
- (is (= #{parent-filename child-filename}
- (dir->file-set (io/file dump-dir "Collection")))
- "Entities go in subdirectories")
- (is (= #{"settings.yaml"}
- (dir->file-set (io/file dump-dir)))
+ (is (= #{[parent-filename (str parent-filename ".yaml")]
+ [parent-filename child-filename (str child-filename ".yaml")]}
+ (file-set (io/file dump-dir "collections")))
+ "collections form a tree, with same-named files")
+ (is (contains? (file-set (io/file dump-dir))
+ ["settings.yaml"])
"A few top-level files are expected"))
(testing "the Collections properly exported"
@@ -41,42 +42,107 @@
(dissoc :id :location)
(assoc :parent_id nil)
(update :created_at t/offset-date-time))
- (-> (yaml/from-file (io/file dump-dir "Collection" parent-filename))
+ (-> (yaml/from-file (io/file dump-dir "collections" parent-filename (str parent-filename ".yaml")))
(update :created_at t/offset-date-time))))
(is (= (-> (into {} (db/select-one Collection :id (:id child)))
(dissoc :id :location)
(assoc :parent_id (:entity_id parent))
(update :created_at t/offset-date-time))
- (-> (yaml/from-file (io/file dump-dir "Collection" child-filename))
+ (-> (yaml/from-file (io/file dump-dir "collections" parent-filename
+ child-filename (str child-filename ".yaml")))
(update :created_at t/offset-date-time))))))))))
+(deftest collection-nesting-test
+ (ts/with-random-dump-dir [dump-dir "serdesv2-"]
+ (ts/with-empty-h2-app-db
+ (ts/with-temp-dpc [Collection [grandparent {:name "Grandparent Collection"
+ :location "/"}]
+ Collection [parent {:name "Parent Collection"
+ :location (str "/" (:id grandparent) "/")}]
+ Collection [child {:name "Child Collection"
+ :location (str "/" (:id grandparent) "/" (:id parent) "/")}]
+ Card [c1 {:name "root card" :collection_id nil}]
+ Card [c2 {:name "grandparent card" :collection_id (:id grandparent)}]
+ Card [c3 {:name "parent card" :collection_id (:id parent)}]
+ Card [c4 {:name "child card" :collection_id (:id child)}]
+ Dashboard [d1 {:name "parent dash" :collection_id (:id parent)}]]
+ (let [export (into [] (extract/extract-metabase nil))]
+ (storage.yaml/store! export dump-dir)
+ (testing "the right files in the right places"
+ (let [gp-dir (str (:entity_id grandparent) "_grandparent_collection")
+ p-dir (str (:entity_id parent) "_parent_collection")
+ c-dir (str (:entity_id child) "_child_collection")]
+ (is (= #{[gp-dir (str gp-dir ".yaml")] ; Grandparent collection
+ [gp-dir p-dir (str p-dir ".yaml")] ; Parent collection
+ [gp-dir p-dir c-dir (str c-dir ".yaml")] ; Child collection
+ ["cards" (str (:entity_id c1) "_root_card.yaml")] ; Root card
+ [gp-dir "cards" (str (:entity_id c2) "_grandparent_card.yaml")] ; Grandparent card
+ [gp-dir p-dir "cards" (str (:entity_id c3) "_parent_card.yaml")] ; Parent card
+ [gp-dir p-dir c-dir "cards" (str (:entity_id c4) "_child_card.yaml")] ; Child card
+ [gp-dir p-dir "dashboards" (str (:entity_id d1) "_parent_dash.yaml")]} ; Parent dashboard
+ (file-set (io/file dump-dir "collections")))))))))))
+
+(deftest snippets-collections-nesting-test
+ (ts/with-random-dump-dir [dump-dir "serdesv2-"]
+ (ts/with-empty-h2-app-db
+ (ts/with-temp-dpc [Collection [grandparent {:name "Grandparent Collection"
+ :namespace :snippets
+ :location "/"}]
+ Collection [parent {:name "Parent Collection"
+ :namespace :snippets
+ :location (str "/" (:id grandparent) "/")}]
+ Collection [child {:name "Child Collection"
+ :namespace :snippets
+ :location (str "/" (:id grandparent) "/" (:id parent) "/")}]
+ NativeQuerySnippet [c1 {:name "root snippet" :collection_id nil}]
+ NativeQuerySnippet [c2 {:name "grandparent snippet" :collection_id (:id grandparent)}]
+ NativeQuerySnippet [c3 {:name "parent snippet" :collection_id (:id parent)}]
+ NativeQuerySnippet [c4 {:name "child snippet" :collection_id (:id child)}]]
+ (let [export (into [] (extract/extract-metabase nil))]
+ (storage.yaml/store! export dump-dir)
+ (let [gp-dir (str (:entity_id grandparent) "_grandparent_collection")
+ p-dir (str (:entity_id parent) "_parent_collection")
+ c-dir (str (:entity_id child) "_child_collection")]
+ (testing "collections under collections/"
+ (is (= #{[gp-dir (str gp-dir ".yaml")] ; Grandparent collection
+ [gp-dir p-dir (str p-dir ".yaml")] ; Parent collection
+ [gp-dir p-dir c-dir (str c-dir ".yaml")]} ; Child collection
+ (file-set (io/file dump-dir "collections")))))
+ (testing "snippets under snippets/"
+ (is (= #{
+ [(str (:entity_id c1) "_root_snippet.yaml")] ; Root snippet
+ [gp-dir (str (:entity_id c2) "_grandparent_snippet.yaml")] ; Grandparent snippet
+ [gp-dir p-dir (str (:entity_id c3) "_parent_snippet.yaml")] ; Parent snippet
+ [gp-dir p-dir c-dir (str (:entity_id c4) "_child_snippet.yaml")]} ; Child snippet
+ (file-set (io/file dump-dir "snippets")))))))))))
+
(deftest embedded-slash-test
(ts/with-random-dump-dir [dump-dir "serdesv2-"]
(ts/with-empty-h2-app-db
(ts/with-temp-dpc [Database [db {:name "My Company Data"}]
Table [table {:name "Customers" :db_id (:id db)}]
Field [website {:name "Company/organization website" :table_id (:id table)}]
- FieldValues [_ {:field_id (:id website)}]]
+ FieldValues [_ {:field_id (:id website)}]
+ Table [_ {:name "Orders/Invoices" :db_id (:id db)}]]
(let [export (into [] (extract/extract-metabase nil))]
(storage.yaml/store! export dump-dir)
(testing "the right files in the right places"
- (is (= #{"Company__SLASH__organization website.yaml"}
- (dir->file-set (io/file dump-dir "Database" "My Company Data" "Table" "Customers" "Field")))
+ (is (= #{["Company__SLASH__organization website.yaml"]
+ ["Company__SLASH__organization website___fieldvalues.yaml"]}
+ (file-set (io/file dump-dir "databases" "My Company Data" "tables" "Customers" "fields")))
"Slashes in file names get escaped")
- (is (= #{"0.yaml"}
- (dir->file-set (io/file dump-dir "Database" "My Company Data" "Table" "Customers"
- "Field" "Company__SLASH__organization website"
- "FieldValues")))
- "Slashes in parent directory names get escaped"))
+ (is (contains? (file-set (io/file dump-dir "databases" "My Company Data" "tables"))
+ ["Orders__SLASH__Invoices" "Orders__SLASH__Invoices.yaml"])
+ "Slashes in directory names get escaped"))
(testing "the Field was properly exported"
(is (= (-> (into {} (serdes.base/extract-one "Field" {} (db/select-one 'Field :id (:id website))))
(update :created_at u.date/format)
(dissoc :serdes/meta))
(-> (yaml/from-file (io/file dump-dir
- "Database" "My Company Data"
- "Table" "Customers"
- "Field" "Company__SLASH__organization website.yaml"))
+ "databases" "My Company Data"
+ "tables" "Customers"
+ "fields" "Company__SLASH__organization website.yaml"))
(update :visibility_type keyword)
(update :base_type keyword))))))))))
diff --git a/src/metabase/models/card.clj b/src/metabase/models/card.clj
index 3fff08e260b..c1d21b051c6 100644
--- a/src/metabase/models/card.clj
+++ b/src/metabase/models/card.clj
@@ -332,10 +332,8 @@
[:name (serdes.hash/hydrated-hash :collection "<none>") :created_at])
;;; ------------------------------------------------- Serialization --------------------------------------------------
-(defmethod serdes.base/extract-query "Card" [_ {:keys [collection-set]}]
- (if (seq collection-set)
- (db/select-reducible Card :collection_id [:in collection-set])
- (db/select-reducible Card)))
+(defmethod serdes.base/extract-query "Card" [_ opts]
+ (serdes.base/extract-query-collections Card opts))
(defn- export-result-metadata [metadata]
(when metadata
@@ -366,16 +364,19 @@
;; :table_id and :database_id are extracted as just :table_id [database_name schema table_name].
;; :collection_id is extracted as its entity_id or identity-hash.
;; :creator_id as the user's email.
- (-> (serdes.base/extract-one-basics "Card" card)
- (update :database_id serdes.util/export-fk-keyed 'Database :name)
- (update :table_id serdes.util/export-table-fk)
- (update :collection_id serdes.util/export-fk 'Collection)
- (update :creator_id serdes.util/export-user)
- (update :made_public_by_id serdes.util/export-user)
- (update :dataset_query serdes.util/export-mbql)
- (update :parameter_mappings serdes.util/export-parameter-mappings)
- (update :visualization_settings serdes.util/export-visualization-settings)
- (update :result_metadata export-result-metadata)))
+ (try
+ (-> (serdes.base/extract-one-basics "Card" card)
+ (update :database_id serdes.util/export-fk-keyed 'Database :name)
+ (update :table_id serdes.util/export-table-fk)
+ (update :collection_id serdes.util/export-fk 'Collection)
+ (update :creator_id serdes.util/export-user)
+ (update :made_public_by_id serdes.util/export-user)
+ (update :dataset_query serdes.util/export-mbql)
+ (update :parameter_mappings serdes.util/export-parameter-mappings)
+ (update :visualization_settings serdes.util/export-visualization-settings)
+ (update :result_metadata export-result-metadata))
+ (catch Exception e
+ (throw (ex-info "Failed to export Card" {:card card} e)))))
(defmethod serdes.base/load-xform "Card"
[card]
@@ -415,3 +416,5 @@
(when (seq template-tags)
(set (for [{:keys [card-id]} template-tags]
["Card" card-id]))))))
+
+(serdes.base/register-ingestion-path! "Card" (serdes.base/ingestion-matcher-collected "collections" "Card"))
diff --git a/src/metabase/models/collection.clj b/src/metabase/models/collection.clj
index e8392f57a06..873360f92dc 100644
--- a/src/metabase/models/collection.clj
+++ b/src/metabase/models/collection.clj
@@ -964,6 +964,21 @@
["Card" card-id]))]
(set/union child-colls dashboards cards)))
+(defmethod serdes.base/storage-path "Collection" [coll {:keys [collections]}]
+ (let [parental (get collections (:entity_id coll))]
+ (concat ["collections"] parental [(last parental)])))
+
+(serdes.base/register-ingestion-path!
+ "Collection"
+ ;; Collections' paths are ["collections" "grandparent" "parent" "me" "me"]
+ (fn [path]
+ (when-let [[id slug] (and (= (first path) "collections")
+ (apply = (take-last 2 path))
+ (serdes.base/split-leaf-file-name (last path)))]
+ (cond-> {:model "Collection" :id id}
+ slug (assoc :label slug)
+ true vector))))
+
;;; +----------------------------------------------------------------------------------------------------------------+
;;; | Perms Checking Helper Fns |
;;; +----------------------------------------------------------------------------------------------------------------+
diff --git a/src/metabase/models/dashboard.clj b/src/metabase/models/dashboard.clj
index bddd277c8fc..1d4b7249ecb 100644
--- a/src/metabase/models/dashboard.clj
+++ b/src/metabase/models/dashboard.clj
@@ -409,11 +409,9 @@
;;; +----------------------------------------------------------------------------------------------------------------+
;;; | SERIALIZATION |
;;; +----------------------------------------------------------------------------------------------------------------+
-(defmethod serdes.base/extract-query "Dashboard" [_ {:keys [collection-set]}]
+(defmethod serdes.base/extract-query "Dashboard" [_ opts]
(eduction (map #(hydrate % :ordered_cards))
- (if (seq collection-set)
- (db/select-reducible Dashboard :collection_id [:in collection-set])
- (db/select-reducible Dashboard))))
+ (serdes.base/extract-query-collections Dashboard opts)))
(defn- extract-dashcard
[dashcard]
@@ -478,3 +476,5 @@
card-id (cond-> (set (keep :card_id parameter_mappings))
card_id (conj card_id))]
["Card" card-id])))
+
+(serdes.base/register-ingestion-path! "Dashboard" (serdes.base/ingestion-matcher-collected "collections" "Dashboard"))
diff --git a/src/metabase/models/dashboard_card.clj b/src/metabase/models/dashboard_card.clj
index 2c5e1a1cc12..db333d93892 100644
--- a/src/metabase/models/dashboard_card.clj
+++ b/src/metabase/models/dashboard_card.clj
@@ -10,6 +10,7 @@
[metabase.models.serialization.hash :as serdes.hash]
[metabase.models.serialization.util :as serdes.util]
[metabase.util :as u]
+ [metabase.util.date-2 :as u.date]
[metabase.util.i18n :refer [tru]]
[metabase.util.schema :as su]
[schema.core :as s]
@@ -233,5 +234,6 @@
(dissoc :serdes/meta)
(update :card_id serdes.util/import-fk 'Card)
(update :dashboard_id serdes.util/import-fk 'Dashboard)
+ (update :created_at #(if (string? %) (u.date/parse %) %))
(update :parameter_mappings serdes.util/import-parameter-mappings)
(update :visualization_settings serdes.util/import-visualization-settings)))
diff --git a/src/metabase/models/database.clj b/src/metabase/models/database.clj
index a13708c7721..fe1791586dd 100644
--- a/src/metabase/models/database.clj
+++ b/src/metabase/models/database.clj
@@ -324,3 +324,16 @@
(m "Database"
(update ingested :details #(or % (:details local) {}))
local)))
+
+(defmethod serdes.base/storage-path "Database" [{:keys [name]} _]
+ ;; ["databases" "db_name" "db_name"] directory for the database with same-named file inside.
+ ["databases" name name])
+
+(serdes.base/register-ingestion-path!
+ "Database"
+ ;; ["databases" "my-db" "my-db"]
+ (fn [[a b c :as path]]
+ (when (and (= (count path) 3)
+ (= a "databases")
+ (= b c))
+ [{:model "Database" :id c}])))
diff --git a/src/metabase/models/field.clj b/src/metabase/models/field.clj
index 53826659ee2..7e3f9270a98 100644
--- a/src/metabase/models/field.clj
+++ b/src/metabase/models/field.clj
@@ -455,3 +455,27 @@
:field_id (:id field)
:serdes/meta [{:model "Dimension" :id (:entity_id dim)}])]
(serdes.base/load-one! dim local)))))
+
+(defmethod serdes.base/storage-path "Field" [field _]
+ (-> field
+ serdes.base/serdes-path
+ drop-last
+ serdes.util/storage-table-path-prefix
+ (concat ["fields" (:name field)])))
+
+(serdes.base/register-ingestion-path!
+ "Field"
+ ;; ["databases" "my-db" "schemas" "PUBLIC" "tables" "customers" "fields" "customer_id"]
+ ;; ["databases" "my-db" "tables" "customers" "fields" "customer_id"]
+ (fn [path]
+ (when-let [{db "databases"
+ schema "schemas"
+ table "tables"
+ field "fields"} (and (#{6 8} (count path))
+ (serdes.base/ingestion-matcher-pairs
+ path [["databases" "schemas" "tables" "fields"]
+ ["databases" "tables" "fields"]]))]
+ (filterv identity [{:model "Database" :id db}
+ (when schema {:model "Schema" :id schema})
+ {:model "Table" :id table}
+ {:model "Field" :id field}]))))
diff --git a/src/metabase/models/field_values.clj b/src/metabase/models/field_values.clj
index 43ad911eb1e..83bb5e36e43 100644
--- a/src/metabase/models/field_values.clj
+++ b/src/metabase/models/field_values.clj
@@ -16,7 +16,8 @@
sandboxed permission try to get values of a Field.
Normally these FieldValues will be deleted after [[advanced-field-values-max-age]] days by the scanning process.
But they will also be automatically deleted when the Full FieldValues of the same Field got updated."
- (:require [clojure.tools.logging :as log]
+ (:require [clojure.string :as str]
+ [clojure.tools.logging :as log]
[java-time :as t]
[metabase.models.serialization.base :as serdes.base]
[metabase.models.serialization.hash :as serdes.hash]
@@ -456,3 +457,34 @@
(= (:type ingested) (:type local)) (dissoc :type)
(= (:hash_key ingested) (:hash_key local)) (dissoc :hash_key))]
((get-method serdes.base/load-update! "") "FieldValues" ingested local)))
+
+(def ^:private field-values-slug "___fieldvalues")
+
+(defmethod serdes.base/storage-path "FieldValues" [fv _]
+ ;; [path to table "fields" "field-name___fieldvalues"] since there's zero or one FieldValues per Field, and Fields
+ ;; don't have their own directories.
+ (let [hierarchy (serdes.base/serdes-path fv)
+ field (last (drop-last hierarchy))
+ table-prefix (serdes.util/storage-table-path-prefix (drop-last 2 hierarchy))]
+ (concat table-prefix
+ ["fields" (str (:id field) field-values-slug)])))
+
+(serdes.base/register-ingestion-path!
+ "FieldValues"
+ ;; ["databases" "my-db" "schemas" "PUBLIC" "tables" "customers" "fields" "customer_id___fieldvalues"]
+ ;; ["databases" "my-db" "tables" "customers" "fields" "customer_id___fieldvalues"]
+ (fn [path]
+ (when-let [{db "databases"
+ schema "schemas"
+ table "tables"
+ field "fields"} (and (#{6 8} (count path))
+ (str/ends-with? (last path) field-values-slug)
+ (serdes.base/ingestion-matcher-pairs
+ path [["databases" "schemas" "tables" "fields"]
+ ["databases" "tables" "fields"]]))]
+ (filterv identity [{:model "Database" :id db}
+ (when schema {:model "Schema" :id schema})
+ {:model "Table" :id table}
+ {:model "Field" :id (subs field 0 (- (count field) (count field-values-slug)))}
+ ;; FieldValues is always just ID 0, since there's at most one as part of the field.
+ {:model "FieldValues" :id "0"}]))))
diff --git a/src/metabase/models/metric.clj b/src/metabase/models/metric.clj
index 8273bbf23ac..9ea0737a8d0 100644
--- a/src/metabase/models/metric.clj
+++ b/src/metabase/models/metric.clj
@@ -94,6 +94,16 @@
(into [] (set/union #{(serdes.util/table->path table_id)}
(serdes.util/mbql-deps definition))))
+(defmethod serdes.base/storage-path "Metric" [metric _ctx]
+ (let [{:keys [id label]} (-> metric serdes.base/serdes-path last)]
+ (-> metric
+ :table_id
+ serdes.util/table->path
+ serdes.util/storage-table-path-prefix
+ (concat ["metrics" (serdes.base/storage-leaf-file-name id label)]))))
+
+(serdes.base/register-ingestion-path! "Metric" (serdes.base/ingestion-matcher-collected "databases" "Metric"))
+
;;; ----------------------------------------------------- OTHER ------------------------------------------------------
(s/defn retrieve-metrics :- [(mi/InstanceOf Metric)]
diff --git a/src/metabase/models/native_query_snippet.clj b/src/metabase/models/native_query_snippet.clj
index b605e810ebb..a64e3a4dbd0 100644
--- a/src/metabase/models/native_query_snippet.clj
+++ b/src/metabase/models/native_query_snippet.clj
@@ -75,10 +75,8 @@
;;; ------------------------------------------------- Serialization --------------------------------------------------
-(defmethod serdes.base/extract-query "NativeQuerySnippet" [_ {:keys [collection-set]}]
- (eduction cat [(db/select-reducible NativeQuerySnippet :collection_id nil)
- (when (seq collection-set)
- (db/select-reducible NativeQuerySnippet :collection_id [:in collection-set]))]))
+(defmethod serdes.base/extract-query "NativeQuerySnippet" [_ opts]
+ (serdes.base/extract-query-collections NativeQuerySnippet opts))
(defmethod serdes.base/extract-one "NativeQuerySnippet"
[_model-name _opts snippet]
@@ -97,3 +95,21 @@
(if collection_id
[[{:model "Collection" :id collection_id}]]
[]))
+
+(defmethod serdes.base/storage-path "NativeQuerySnippet" [snippet ctx]
+ ;; Intended path here is ["snippets" "nested" "collections" "snippet_eid_and_slug"]
+ ;; We just the default path, then pull it apart.
+ ;; The default is ["collections" "nested" collections" "nativequerysnippets" "base_name"]
+ (let [basis (serdes.base/storage-default-collection-path snippet ctx)
+ file (last basis)
+ colls (->> basis rest (drop-last 2))] ; Drops the "collections" at the start, and the last two.
+ (concat ["snippets"] colls [file])))
+
+(serdes.base/register-ingestion-path!
+ "NativeQuerySnippet"
+ (fn [path]
+ (when-let [[id slug] (and (= (first path) "snippets")
+ (serdes.base/split-leaf-file-name (last path)))]
+ (cond-> {:model "NativeQuerySnippet" :id id}
+ slug (assoc :label slug)
+ true vector))))
diff --git a/src/metabase/models/segment.clj b/src/metabase/models/segment.clj
index 0f446e9541c..48aa9a316f8 100644
--- a/src/metabase/models/segment.clj
+++ b/src/metabase/models/segment.clj
@@ -94,6 +94,16 @@
(into [] (set/union #{(serdes.util/table->path table_id)}
(serdes.util/mbql-deps definition))))
+(defmethod serdes.base/storage-path "Segment" [segment _ctx]
+ (let [{:keys [id label]} (-> segment serdes.base/serdes-path last)]
+ (-> segment
+ :table_id
+ serdes.util/table->path
+ serdes.util/storage-table-path-prefix
+ (concat ["segments" (serdes.base/storage-leaf-file-name id label)]))))
+
+(serdes.base/register-ingestion-path! "Segment" (serdes.base/ingestion-matcher-collected "databases" "Segment"))
+
;;; ------------------------------------------------------ Etc. ------------------------------------------------------
(s/defn retrieve-segments :- [(mi/InstanceOf Segment)]
diff --git a/src/metabase/models/serialization/base.clj b/src/metabase/models/serialization/base.clj
index d2fac6c0ab8..74b3f7935f3 100644
--- a/src/metabase/models/serialization/base.clj
+++ b/src/metabase/models/serialization/base.clj
@@ -9,7 +9,8 @@
If the model is not exported, add it to the exclusion lists in the tests. Every model should be explicitly listed as
exported or not, and a test enforces this so serialization isn't forgotten for new models."
- (:require [clojure.tools.logging :as log]
+ (:require [clojure.string :as str]
+ [clojure.tools.logging :as log]
[metabase.models.interface :as mi]
[metabase.models.serialization.hash :as serdes.hash]
[metabase.util :as u]
@@ -152,6 +153,18 @@
;;; The storage system might transform that stream in some arbitrary way. Storage is a dead end - it should perform side
;;; effects like writing to the disk or network, and return nothing.
;;;
+;;; Not all storage solutions use directory structure, but for those that do, [[storage-path]] should give the path as
+;;; a list of strings: `["foo" "bar" "some_file"]`. Note the lack of a file extension on the last segment - that
+;;; is deliberately left off so that no filename surgery is required to support eg. both JSON and YAML output.
+;;;
+;;; By convention, models are named as plural and in lower case:
+;;; `["collections" "1234ABC_my_collection" "dashboards" "8765def_health_metrics"]`.
+;;;
+;;; As a final remark, note that some entities have their own directories and some do not. For example a Field is
+;;; simply a file, while a Table has a directory. So a Table's itself is
+;;; `["databases" "some-db" "schemas" "PUBLIC" "tables" "Customer" "Customer"]`
+;;; so that's a directory called `Customer` with a file called (for YAML output) `Customer.yaml` in it.
+;;;
;;; Selective Serialization:
;;; Sometimes we want to export a "subtree" instead of the complete appdb. At the simplest, we might serialize a single
;;; question. Moving up, it might be a Dashboard and all its questions, or a Collection and all its content Cards and
@@ -211,6 +224,19 @@
(eduction (map (partial extract-one model opts))
(extract-query model opts)))
+(defn extract-query-collections
+ "Helper for the common (but not default) [[extract-query]] case of fetching everything that isn't in a personal
+ collection."
+ [model {:keys [collection-set]}]
+ (if collection-set
+ ;; If collection-set is defined, select everything in those collections, or with nil :collection_id.
+ (let [in-colls (db/select-reducible model :collection_id [:in collection-set])]
+ (if (contains? collection-set nil)
+ (eduction cat [in-colls (db/select-reducible model :collection_id nil)])
+ in-colls))
+ ;; If collection-set is nil, just select everything.
+ (db/select-reducible model)))
+
(defmethod extract-query :default [model-name _]
(db/select-reducible (symbol model-name)))
@@ -472,3 +498,136 @@
(if (entity-id? id-str)
(db/select-one model :entity_id id-str)
(find-by-identity-hash model id-str)))
+
+;;; +----------------------------------------------------------------------------------------------------------------+
+;;; | Storage |
+;;; +----------------------------------------------------------------------------------------------------------------+
+;;; These storage multimethods take a second argument known as the context. This is a good place for a particular
+;;; storage implementation to include some precomputed information, or options.
+;;; In particular, it should include a table of collection IDs to path fragments that is precomputed by the host.
+;;; [[storage-base-context]] computes that, since many things go in a collections tree.
+(def ^:private max-label-length 100)
+
+(defn- truncate-label [s]
+ (if (> (count s) max-label-length)
+ (subs s 0 max-label-length)
+ s))
+
+(defn- lower-plural [s]
+ (-> s u/lower-case-en (str "s")))
+
+(defn storage-leaf-file-name
+ "Captures the common pattern for leaf file names as `entityID_label`."
+ ([id] (str id))
+ ([id label] (if (nil? label)
+ (storage-leaf-file-name id)
+ (str id "_" (truncate-label label)))))
+
+(defn storage-default-collection-path
+ "Implements the most common structure for [[storage-path]] - `collections/c1/c2/c3/models/entityid_slug.ext`"
+ [entity {:keys [collections]}]
+ (let [{:keys [model id label]} (-> entity serdes-path last)]
+ (concat ["collections"]
+ (get collections (:collection_id entity)) ;; This can be nil, but that's fine - that's the root collection.
+ [(lower-plural model) (storage-leaf-file-name id label)])))
+
+(defmulti storage-path
+ "Computes the complete storage path for a given entity.
+ `(storage-path entity ctx)`
+ Dispatches on the model name, eg. \"Dashboard\".
+
+ Returns a list of strings giving the path, with the final entry being the file name with no extension.
+
+ The default implementation works for entities which are:
+ - Part of the regular (not snippet) collections tree, per a :collection_id field; and
+ - Stored as `foos/1234abc_slug.extension` underneath their collection
+ eg. Cards, Dashboards, Timelines
+
+ The default logic is captured by [[storage-default-collection-path]] so it can be reused."
+ {:arglists '([entity ctx])}
+ (fn [entity _] (ingested-model entity)))
+
+(defmethod storage-path :default [entity ctx]
+ (storage-default-collection-path entity ctx))
+
+(defn storage-base-context
+ "Creates the basic context for storage. This is a map with a single entry: `:collections` is a map from collection ID
+ to the path of collections."
+ []
+ (let [colls (db/select ['Collection :id :entity_id :location :slug])
+ coll-names (into {} (for [{:keys [id entity_id slug]} colls]
+ [(str id) (storage-leaf-file-name entity_id slug)]))
+ coll->path (into {} (for [{:keys [entity_id id location]} colls
+ :let [parents (rest (str/split location #"/"))]]
+ [entity_id (map coll-names (concat parents [(str id)]))]))]
+ {:collections coll->path}))
+
+;;; +----------------------------------------------------------------------------------------------------------------+
+;;; | Ingestion |
+;;; +----------------------------------------------------------------------------------------------------------------+
+(defonce ^:private ingest-path-matchers (atom {}))
+
+(defn ingest-path
+ "Transforms a file path (as a sequence of strings with no file extension on the last part) into a `:serdes/meta`
+ hierarchy, or nil if nothing matches.
+
+ The set of matchers is maintained in an atom, and each model should call [[register-ingestion-path!]] to add its
+ matcher.
+
+ Note that the input format is the same as the return value of [[storage-path]]."
+ [path]
+ (first (keep #(% path) (vals @ingest-path-matchers))))
+
+(defn register-ingestion-path!
+ "Registers the matcher for the given model. Expects the model name to be a string.
+ The matcher is a function from a path (see [[ingest-path]] for the format) to a `:serdes/meta` hierarchy, or nil
+ if there's no match."
+ [model-name matcher]
+ (swap! ingest-path-matchers assoc model-name matcher)
+ ;; Return a readable symbol so this call shows up nicely when (a buffer containing) a register-ingestion-path! call
+ ;; is evaluated at the REPL.
+ ['register-ingestion-path! model-name])
+
+(defn split-leaf-file-name
+ "Given a leaf file name of the type generated by [[storage-leaf-file-name]], break it apart into an [id slug] pair,
+ where the slug might be nil."
+ [file]
+ (when-let [[_ id slug] (or (re-matches #"^([A-Za-z0-9_\.:-]{21})_(.*)$" file) ; entity_id and slug
+ (re-matches #"^([A-Za-z0-9_\.:-]{21})$" file) ; entity_id only
+ (re-matches #"^([a-fA-F0-9]{8})_(.*)$" file) ; Hash and slug
+ (re-matches #"^([a-fA-F0-9]{8})$" file))] ; Hash only
+ [id slug]))
+
+(defn ingestion-matcher-collected
+ "A helper for the common case of paths like `collections/some/nested/collections/model-name/entityID_slug`.
+ Expects the (lowercase) first segment, and the model name (eg. \"Dashboard\", not \"dashboards\" as it appears in
+ the path).
+ For example `(ingestion-matcher-collected \"collections\" \"Card\")`.
+
+ Returns a matcher function.
+ The resulting hierarchy is "
+ [first-segment model-name]
+ (fn [path]
+ (let [head (first path)
+ [model file] (take-last 2 path)]
+ (when-let [[id slug] (and (= head first-segment)
+ (= model (lower-plural model-name))
+ (split-leaf-file-name file))]
+ (cond-> {:model model-name :id id}
+ slug (assoc :label slug)
+ true vector)))))
+
+(defn- match-pairs [path pattern]
+ (let [chunks (take (count pattern) (partition 2 path))]
+ (when (= pattern (map first chunks))
+ (reduce (fn [out [k v]] (assoc out k v)) {} chunks))))
+
+(defn ingestion-matcher-pairs
+ "A helper for the common case of paths like `databases/my-db/schemas/my-schema/tables/my-table` which alternate
+ fixed and arbitrary segments.
+ The input is a *list* of sequences like `[[\"databases\" \"schemas\" \"tables\"] [\"databases\" \"tables\"]]` and the
+ response is a map of those fixed tokens as keys to the following segment as values, eg.
+ `{\"databases\" \"my-db\" \"schemas\" \"my-schema\" \"tables\" \"my-table\"}`.
+ This matches a *prefix*, not necessarily the entire sequence."
+ [path patterns]
+ (some (partial match-pairs path) patterns))
diff --git a/src/metabase/models/serialization/util.clj b/src/metabase/models/serialization/util.clj
index 2486ffab2a8..426eeeab7bb 100644
--- a/src/metabase/models/serialization/util.clj
+++ b/src/metabase/models/serialization/util.clj
@@ -124,6 +124,23 @@
(when schema {:model "Schema" :id schema})
{:model "Table" :id table-name}]))
+(defn storage-table-path-prefix
+ "The [[serdes.base/storage-path]] for Table is a bit tricky, and shared with Fields and FieldValues, so it's
+ factored out here.
+ Takes the :serdes/meta value for a `Table`!
+ The return value includes the directory for the Table, but not the file for the Table itself.
+
+ With a schema: `[\"databases\" \"db_name\" \"schemas\" \"public\" \"tables\" \"customers\"]`
+ No schema: `[\"databases\" \"db_name\" \"tables\" \"customers\"]`"
+ [path]
+ (let [db-name (-> path first :id)
+ schema (when (= (count path) 3)
+ (-> path second :id))
+ table-name (-> path last :id)]
+ (concat ["databases" db-name]
+ (when schema ["schemas" schema])
+ ["tables" table-name])))
+
;; -------------------------------------------------- Fields ---------------------------------------------------------
(defn export-field-fk
"Given a numeric `field_id`, return a portable field reference.
@@ -328,6 +345,10 @@
["field" (field :guard vector?) tail] (into #{(field->path field)} (mbql-deps-map tail))
[:field-id (field :guard vector?) tail] (into #{(field->path field)} (mbql-deps-map tail))
["field-id" (field :guard vector?) tail] (into #{(field->path field)} (mbql-deps-map tail))
+ [:metric (field :guard portable-id?)] #{[{:model "Metric" :id field}]}
+ ["metric" (field :guard portable-id?)] #{[{:model "Metric" :id field}]}
+ [:segment (field :guard portable-id?)] #{[{:model "Segment" :id field}]}
+ ["segment" (field :guard portable-id?)] #{[{:model "Segment" :id field}]}
:else (reduce #(cond
(map? %2) (into %1 (mbql-deps-map %2))
(vector? %2) (into %1 (mbql-deps-vector %2))
diff --git a/src/metabase/models/table.clj b/src/metabase/models/table.clj
index f4dba6eb19e..874d43bd241 100644
--- a/src/metabase/models/table.clj
+++ b/src/metabase/models/table.clj
@@ -13,6 +13,7 @@
[metabase.models.segment :refer [retrieve-segments Segment]]
[metabase.models.serialization.base :as serdes.base]
[metabase.models.serialization.hash :as serdes.hash]
+ [metabase.models.serialization.util :as serdes.util]
[metabase.util :as u]
[toucan.db :as db]
[toucan.models :as models]))
@@ -264,3 +265,23 @@
[{:keys [db_id] :as table}]
(-> (serdes.base/load-xform-basics table)
(assoc :db_id (db/select-one-field :id 'Database :name db_id))))
+
+(defmethod serdes.base/storage-path "Table" [table _ctx]
+ (concat (serdes.util/storage-table-path-prefix (serdes.base/serdes-path table))
+ [(:name table)]))
+
+(serdes.base/register-ingestion-path!
+ "Table"
+ ;; ["databases" "my-db" "schemas" "PUBLIC" "tables" "customers" "customers"]
+ ;; ["databases" "my-db" "tables" "customers" "customers"]
+ ;; Note that the last 2 must match, they're the table's directory and its file.
+ (fn [path]
+ (when-let [{db "databases"
+ schema "schemas"
+ table "tables"} (and (#{5 7} (count path))
+ (apply = (take-last 2 path))
+ (serdes.base/ingestion-matcher-pairs path [["databases" "schemas" "tables"]
+ ["databases" "tables"]]))]
+ (filterv identity [{:model "Database" :id db}
+ (when schema {:model "Schema" :id schema})
+ {:model "Table" :id table}]))))
diff --git a/src/metabase/models/timeline.clj b/src/metabase/models/timeline.clj
index 18b06aec80e..8ec5de88a76 100644
--- a/src/metabase/models/timeline.clj
+++ b/src/metabase/models/timeline.clj
@@ -65,17 +65,15 @@
[:name (serdes.hash/hydrated-hash :collection "<none>") :created_at])
;;;; serialization
-(defmethod serdes.base/extract-query "Timeline" [_model-name {:keys [collection-set]}]
+(defmethod serdes.base/extract-query "Timeline" [_model-name opts]
(eduction (map #(timeline-event/include-events-singular % {:all? true}))
- (if (seq collection-set)
- (db/select-reducible Timeline :collection_id [:in collection-set])
- (db/select-reducible Timeline))))
+ (serdes.base/extract-query-collections Timeline opts)))
(defn- extract-events [events]
(sort-by :timestamp
(for [event events]
(-> (into (sorted-map) event)
- (dissoc :creator :id :timeline_id)
+ (dissoc :creator :id :timeline_id :updated_at)
(update :creator_id serdes.util/export-user)
(update :timestamp #(u.date/format (t/offset-date-time %)))))))
@@ -107,3 +105,5 @@
(defmethod serdes.base/serdes-dependencies "Timeline" [{:keys [collection_id]}]
[[{:model "Collection" :id collection_id}]])
+
+(serdes.base/register-ingestion-path! "Timeline" (serdes.base/ingestion-matcher-collected "collections" "Timeline"))
diff --git a/src/metabase/models/timeline_event.clj b/src/metabase/models/timeline_event.clj
index 81f258af1e4..2d19e37a914 100644
--- a/src/metabase/models/timeline_event.clj
+++ b/src/metabase/models/timeline_event.clj
@@ -110,4 +110,5 @@
serdes.base/load-xform-basics
(update :timeline_id serdes.util/import-fk 'Timeline)
(update :creator_id serdes.util/import-user)
- (update :timestamp u.date/parse)))
+ (update :timestamp u.date/parse)
+ (update :created_at #(if (string? %) (u.date/parse %) %))))
--
GitLab