`POST /api/ee/serialization/serialize/data-model` endpoint (#25640)

* `POST /api.ee/serialization/serialize/data-model` endpoint

* Sort namespaces

* Don't use `tru` for endpoint params validation

.clj-kondo/config.edn

@@ -416,6 +416,7 @@
   metabase.test.data.interface/defdataset-edn                                          clojure.core/def
   metabase.test/defdataset                                                             clojure.core/def
   metabase.test/with-open-channels                                                     clojure.core/let
+  metabase.test/with-temp-dir                                                          clojure.core/let
   metabase.test/with-temp-file                                                         clojure.core/let
   metabase.test/with-user-in-groups                                                    clojure.core/let
   metabase.util.files/with-open-path-to-resource                                       clojure.core/let

enterprise/backend/src/metabase_enterprise/api/routes.clj

@@ -4,12 +4,16 @@
 
   These routes should generally live under prefixes like `/api/ee/<feature>/` -- see the
   `enterprise/backend/README.md` for more details."
-  (:require [compojure.core :as compojure]
-            [metabase-enterprise.advanced-permissions.api.routes :as advanced-permissions]
-            [metabase-enterprise.api.routes.common :as ee.api.common]
-            [metabase-enterprise.audit-app.api.routes :as audit-app]
-            [metabase-enterprise.content-management.api.routes :as content-management]
-            [metabase-enterprise.sandbox.api.routes :as sandbox]))
+  (:require
+   [compojure.core :as compojure]
+   [metabase-enterprise.advanced-permissions.api.routes
+    :as advanced-permissions]
+   [metabase-enterprise.api.routes.common :as ee.api.common]
+   [metabase-enterprise.audit-app.api.routes :as audit-app]
+   [metabase-enterprise.content-management.api.routes
+    :as content-management]
+   [metabase-enterprise.sandbox.api.routes :as sandbox]
+   [metabase-enterprise.serialization.api.routes :as serialization]))
 
 (compojure/defroutes ^{:doc "API routes only available when running Metabase® Enterprise Edition™."} routes
   ;; The following routes are NAUGHTY and do not follow the naming convention (i.e., they do not start with
@@ -27,4 +31,7 @@
     (ee.api.common/+require-premium-feature :audit-app audit-app/routes))
    (compojure/context
     "/advanced-permissions" []
-    (ee.api.common/+require-premium-feature :advanced-permissions advanced-permissions/routes))))
+    (ee.api.common/+require-premium-feature :advanced-permissions advanced-permissions/routes))
+   (compojure/context
+    "/serialization" []
+    (ee.api.common/+require-premium-feature :serialization serialization/routes))))

enterprise/backend/src/metabase_enterprise/serialization/api/routes.clj

+(ns metabase-enterprise.serialization.api.routes
+  "/api/ee/serialization/ routes"
+  (:require
+   [compojure.core :as compojure]
+   [metabase-enterprise.serialization.api.serialize
+    :as
+    ee.api.serialization.serialize]))
+
+;;; all these routes require the `:serialization` premium feature; this is done
+;;; in [[metabase-enterprise.api.routes/routes]]
+(compojure/defroutes ^{:doc "Routes for serialization endpoints."} routes
+  (compojure/context
+   "/serialize"
+   []
+   ee.api.serialization.serialize/routes))

enterprise/backend/src/metabase_enterprise/serialization/api/serialize.clj

+(ns metabase-enterprise.serialization.api.serialize
+  "/api/ee/serialization/serialize endpoints"
+  (:require
+   [clojure.set :as set]
+   [compojure.core :as compojure :refer [POST]]
+   [metabase-enterprise.serialization.cmd :as serialization.cmd]
+   [metabase.api.common :as api]
+   [metabase.models.collection :refer [Collection]]
+   [metabase.util.i18n :refer [tru]]
+   [metabase.util.schema :as su]
+   [toucan.db :as db]))
+
+(api/defendpoint POST "/data-model"
+  "This endpoint should serialize: the data model, settings.yaml, and all the selected Collections
+
+  The data model should only change if the user triggers a manual sync or scan (since the scheduler is turned off)
+
+  The user will need to add somewhere (probably in the admin panel):
+
+  - A path (maybe we can assume it will always dump to the same path as the Metabase jar, but we probably want to let
+    them define the path)
+
+  - The collections that they want to serialize (using selective serialization)"
+  [:as {{:keys [collection_ids path]} :body}]
+  {collection_ids (su/with-api-error-message
+                    (su/distinct (su/non-empty [su/IntGreaterThanZero]))
+                    "Non-empty, distinct array of Collection IDs")
+   path           (su/with-api-error-message su/NonBlankString
+                    "Valid directory to serialize results to")}
+  ;; Make sure all the specified collection IDs exist.
+  (let [existing-collection-ids (db/select-ids Collection :id [:in (set collection_ids)])]
+    (when-not (= (set collection_ids) (set existing-collection-ids))
+      (throw (ex-info (tru "Invalid Collection ID(s). These Collections do not exist: {0}"
+                           (pr-str (set/difference (set collection_ids) (set existing-collection-ids))))
+                      {:status-code 404}))))
+  (serialization.cmd/dump path
+                          {:v2      true
+                           :targets (for [collection-id collection_ids]
+                                      ["Collection" collection-id])})
+  ;; TODO -- not 100% sure this response makes sense. We can change it later with something more meaningful maybe
+  {:status :ok})
+
+(api/define-routes
+  ;; for now let's say you have to be an admin to hit any of the serialization endpoints
+  api/+check-superuser)

enterprise/backend/src/metabase_enterprise/serialization/cmd.clj

@@ -42,7 +42,7 @@
     (deferred-trs "invalid context seed value")))
 
 (s/defn v1-load
-  "Load serialized metabase instance as created by `dump` command from directory `path`."
+  "Load serialized metabase instance as created by [[dump]] command from directory `path`."
   [path context :- Context]
   (plugins/load-plugins!)
   (mdb/setup-db!)
@@ -176,10 +176,16 @@
   (log/info (trs "END DUMP to {0} via user {1}" path user)))
 
 (defn- v2-extract [opts]
-  (if (:collections opts)
-    (v2.extract/extract-subtrees (assoc opts :targets (for [c (str/split (:collections opts) #",")]
-                                                        ["Collection" (Integer/parseInt c)])))
-    (v2.extract/extract-metabase opts)))
+  ;; if opts has `collections` (a comma-separated string) then convert those to a list of `:targets`
+  (let [opts (cond-> opts
+               (:collections opts)
+               (assoc :targets (for [c (str/split (:collections opts) #",")]
+                                 ["Collection" (Integer/parseInt c)])))]
+    ;; if we have `:targets` (either because we created them from `:collections`, or because they were specified
+    ;; elsewhere) use [[v2.extract/extract-subtrees]]
+    (if (:targets opts)
+      (v2.extract/extract-subtrees opts)
+      (v2.extract/extract-metabase opts))))
 
 (defn- v2-dump [path opts]
   (-> (v2-extract opts)
@@ -190,7 +196,9 @@
   [path {:keys [state user v2]
          :or {state :active}
          :as opts}]
-  (mdb/setup-db!) (db/select 'User)
+  (log/tracef "Dumping to %s with options %s" (pr-str path) (pr-str opts))
+  (mdb/setup-db!)
+  (db/select User) ;; TODO -- why???
   (if v2
     (v2-dump path opts)
     (v1-dump path state user opts)))

enterprise/backend/src/metabase_enterprise/serialization/v2/extract.clj

@@ -2,10 +2,12 @@
   "Extraction is the first step in serializing a Metabase appdb so it can be eg. written to disk.
 
   See the detailed descriptions of the (de)serialization processes in [[metabase.models.serialization.base]]."
-  (:require [clojure.set :as set]
-            [medley.core :as m]
-            [metabase-enterprise.serialization.v2.models :as serdes.models]
-            [metabase.models.serialization.base :as serdes.base]))
+  (:require
+   [clojure.set :as set]
+   [clojure.tools.logging :as log]
+   [medley.core :as m]
+   [metabase-enterprise.serialization.v2.models :as serdes.models]
+   [metabase.models.serialization.base :as serdes.base]))
 
 (defn extract-metabase
   "Extracts the appdb into a reducible stream of serializable maps, with `:serdes/meta` keys.
@@ -16,6 +18,7 @@
   Takes an options map which is passed on to [[serdes.base/extract-all]] for each model. The options are documented
   there."
   [opts]
+  (log/tracef "Extracting Metabase with options: %s" (pr-str opts))
   (let [model-pred (if (:data-model-only opts)
                      #{"Database" "Dimension" "Field" "FieldValues" "Metric" "Segment" "Table"}
                      (constantly true))]
@@ -43,10 +46,11 @@
   complete transitive closure of all descendants is found. This produces a set of `[\"ModelName\" id]` pairs, which
   entities are then extracted the same way as [[extract-metabase]]."
   [{:keys [targets] :as opts}]
+  (log/tracef "Extracting subtrees with options: %s" (pr-str opts))
   (let [closure  (descendants-closure targets)
         by-model (->> closure
-                     (group-by first)
-                     (m/map-vals #(set (map second %))))]
+                      (group-by first)
+                      (m/map-vals #(set (map second %))))]
     (eduction cat (for [[model ids] by-model]
                     (eduction (map #(serdes.base/extract-one model opts %))
                               (serdes.base/raw-reducible-query model {:where [:in :id ids]}))))))

enterprise/backend/test/metabase_enterprise/serialization/api/serialize_test.clj

+(ns metabase-enterprise.serialization.api.serialize-test
+  (:require
+   [clojure.test :refer :all]
+   [metabase.models :refer [Card Collection Dashboard DashboardCard]]
+   [metabase.public-settings.premium-features-test
+    :as premium-features-test]
+   [metabase.test :as mt]
+   [metabase.util.files :as u.files]
+   [toucan.db :as db]))
+
+(defn- do-serialize-data-model [f]
+  (premium-features-test/with-premium-features #{:serialization}
+    (mt/with-temp* [Collection    [{collection-id :id}]
+                    Dashboard     [{dashboard-id :id} {:collection_id collection-id}]
+                    Card          [{card-id :id}      {:collection_id collection-id}]
+                    DashboardCard [_                  {:card_id card-id, :dashboard_id dashboard-id}]]
+      (testing "Sanity Check"
+        (is (integer? collection-id))
+        (is (= collection-id
+               (db/select-one-field :collection_id Card :id card-id))))
+      (mt/with-temp-dir [dir "serdes-dir"]
+        (f {:collection-id collection-id, :dir dir})))))
+
+(deftest serialize-data-model-happy-path-test
+  (do-serialize-data-model
+   (fn [{:keys [collection-id dir]}]
+     (is (= {:status "ok"}
+            (mt/user-http-request :crowberto :post 200 "ee/serialization/serialize/data-model"
+                                  {:collection_ids [collection-id]
+                                   :path           dir})))
+     (testing "Created files"
+       (letfn [(path-files [path]
+                 (sort (map str (u.files/files-seq path))))
+               (files [& path-components]
+                 (path-files (apply u.files/get-path dir path-components)))]
+         (is (= ["/tmp/serdes-dir/Card"
+                 "/tmp/serdes-dir/Collection"
+                 "/tmp/serdes-dir/Dashboard"
+                 "/tmp/serdes-dir/settings.yaml"]
+                (files)))
+         (testing "subdirs"
+           (testing "Card"
+             (is (= 1
+                    (count (files "Card")))))
+           (testing "Collection"
+             (is (= 1
+                    (count (files "Collection")))))
+           (testing "Dashboard"
+             (is (= 2
+                    (count (files "Dashboard"))))
+             (let [[f1 f2]         (files "Dashboard")
+                   [path-1 path-2] (map u.files/get-path [f1 f2])]
+               (testing "Should have one subdirectory"
+                 (is (= 1
+                        (count (filter true? (map u.files/regular-file? [path-1 path-2]))))))
+               (let [subdirectory-path (first (remove u.files/regular-file? [path-1 path-2]))]
+                 (is (= 1
+                        (count (path-files subdirectory-path)))))))))))))
+
+(deftest serialize-data-model-validation-test
+  (do-serialize-data-model
+   (fn [{:keys [collection-id dir]}]
+     (let [good-request {:collection_ids [collection-id]
+                         :path           dir}
+           serialize!   (fn [& {:keys [expected-status-code
+                                       request
+                                       user]
+                                :or   {expected-status-code 400
+                                       request              good-request
+                                       user                 :crowberto}}]
+                          (mt/user-http-request user :post expected-status-code "ee/serialization/serialize/data-model"
+                                                request))]
+       (testing "Require a EE token with the `:serialization` feature"
+         (premium-features-test/with-premium-features #{}
+           (is (= "This API endpoint is only enabled if you have a premium token with the :serialization feature."
+                  (serialize! :expected-status-code 402)))))
+       (testing "Require current user to be a superuser"
+         (is (= "You don't have permissions to do that."
+                (serialize! :user :rasta, :expected-status-code 403))))
+       (testing "Require valid collection_ids"
+         (testing "Non-empty"
+           (is (= {:errors {:collection_ids "Non-empty, distinct array of Collection IDs"}}
+                  (serialize! :request (dissoc good-request :collection_ids))
+                  (serialize! :request (assoc good-request :collection_ids nil))
+                  (serialize! :request (assoc good-request :collection_ids [])))))
+         (testing "No duplicates"
+           (is (= {:errors {:collection_ids "Non-empty, distinct array of Collection IDs"}}
+                  (serialize! :request (assoc good-request :collection_ids [collection-id collection-id])))))
+         (testing "All Collections must exist"
+           (is (= (format "Invalid Collection ID(s). These Collections do not exist: #{%d}" Integer/MAX_VALUE)
+                  (serialize! :request (assoc good-request :collection_ids [collection-id Integer/MAX_VALUE])
+                              :expected-status-code 404))))
+         (testing "Invalid value"
+           (is (= {:errors {:collection_ids "Non-empty, distinct array of Collection IDs"}}
+                  (serialize! :request (assoc good-request :collection_ids collection-id))
+                  (serialize! :request (assoc good-request :collection_ids "My Collection"))))))
+       (testing "Validate 'path' parameter"
+         (is (= {:errors {:path "Valid directory to serialize results to"}}
+                (serialize! :request (dissoc good-request :path))
+                (serialize! :request (assoc good-request :path ""))
+                (serialize! :request (assoc good-request :path 1000)))))))))

src/metabase/models/serialization/hash.clj

@@ -18,8 +18,10 @@
     - NO numeric database IDs!
     - Any foreign keys should be hydrated and the identity-hash of the foreign entity used as part of the hash.
       - There's a [[hydrated-hash]] helper for this with several example uses."
-  (:require [potemkin.types :as p.types]
-            [toucan.hydrate :refer [hydrate]]))
+  (:require
+   [metabase.util.i18n :refer [tru]]
+   [potemkin.types :as p.types]
+   [toucan.hydrate :refer [hydrate]]))
 
 ;;; +----------------------------------------------------------------------------------------------------------------+
 ;;; |                                              Identity Hashes                                                   |
@@ -44,7 +46,14 @@
   "Hashes a Clojure value into an 8-character hex string, which is used as the identity hash.
   Don't call this outside a test, use [[identity-hash]] instead."
   [target]
-  (format "%08x" (hash target)))
+  (when (sequential? target)
+    (assert (seq target) "target cannot be an empty sequence"))
+  (try
+    (format "%08x" (hash target))
+    (catch Throwable e
+      (throw (ex-info (tru "Error calculating raw hash: {0}" (ex-message e))
+                      {:target target}
+                      e)))))
 
 (defn identity-hash
   "Given a modeled entity, return its identity hash for use in serialization. The hash is an 8-character hex string.
@@ -52,16 +61,26 @@
   These hashes are intended to be a decently robust fallback for older entities whose `entity_id` fields are not
   populated."
   [entity]
-  (-> (for [f (identity-hash-fields entity)]
-        (f entity))
-      raw-hash))
+  {:pre [(some? entity)]}
+  (try
+    (-> (for [f (identity-hash-fields entity)]
+          (f entity))
+        raw-hash)
+    (catch Throwable e
+      (throw (ex-info (tru "Error calculating identity hash: {0}" (ex-message e))
+                      {:entity entity}
+                      e)))))
 
 (defn hydrated-hash
   "Many entities reference other entities. Using the autoincrementing ID is not portable, so we use the identity hash
   of the referenced entity. This is a helper for writing [[identity-hash-fields]]."
   [hydration-key]
   (fn [entity]
-    (-> entity
-        (hydrate hydration-key)
-        (get hydration-key)
-        identity-hash)))
+    (let [hydrated-value (get (hydrate entity hydration-key) hydration-key)]
+      (when (nil? hydrated-value)
+        (throw (ex-info (tru "Error calculating hydrated hash: {0} is nil after hydrating {1}"
+                             (pr-str hydration-key)
+                             (name entity))
+                        {:entity        entity
+                         :hydration-key hydration-key})))
+      (identity-hash hydrated-value))))

src/metabase/public_settings/premium_features.clj

@@ -237,6 +237,10 @@
   etc.)?"
   :content-management)
 
+(define-premium-feature ^{:added "0.45.0"} enable-serialization?
+  "Enable the v2 SerDes functionality"
+  :serialization)
+
 (defsetting is-hosted?
   "Is the Metabase instance running in the cloud?"
   :type       :boolean

test/metabase/test.clj

@@ -211,6 +211,7 @@
   with-non-admin-groups-no-root-collection-for-namespace-perms
   with-non-admin-groups-no-root-collection-perms
   with-temp-env-var-value
+  with-temp-dir
   with-temp-file
   with-temp-scheduler
   with-temp-vals-in-db

test/metabase/test/util.clj

@@ -1054,11 +1054,13 @@
         (io/delete-file (io/file filename) :silently)))))
 
 (defmacro with-temp-file
-  "Execute `body` with newly created temporary file(s) in the system temporary directory. You may optionally specify the
+  "Execute `body` with a path for temporary file(s) in the system temporary directory. You may optionally specify the
   `filename` (without directory components) to be created in the temp directory; if `filename` is nil, a random
   filename will be used. The file will be deleted if it already exists, but will not be touched; use `spit` to load
   something in to it.
 
+  DOES NOT CREATE A FILE!
+
     ;; create a random temp filename. File is deleted if it already exists.
     (with-temp-file [filename]
       ...)
@@ -1118,6 +1120,24 @@
       `(with-temp-file [])
       `(with-temp-file (+ 1 2)))))
 
+(defn do-with-temp-dir
+  "Impl for [[with-temp-dir]] macro."
+  [temp-dir-name f]
+  (do-with-temp-file
+   temp-dir-name
+   (^:once fn* [path]
+    (let [file (io/file path)]
+      (when (.exists file)
+        (org.apache.commons.io.FileUtils/deleteDirectory file)))
+    (u.files/create-dir-if-not-exists! (u.files/get-path path))
+    (f path))))
+
+(defmacro with-temp-dir
+  "Like [[with-temp-file]], but creates a new temporary directory in the system temp dir. Deletes existing directory if
+  it already exists."
+  [[directory-binding dir-name] & body]
+  `(do-with-temp-dir ~dir-name (^:once fn* [~directory-binding] ~@body)))
+
 (defn do-with-user-in-groups
   ([f groups-or-ids]
    (tt/with-temp User [user]

test/metabase/util/files_test.clj

+(ns metabase.util.files-test
+  (:require
+   [clojure.test :refer :all]
+   [metabase.test :as mt]
+   [metabase.util.files :as u.files]))
+
+(deftest is-regular-file-test
+  (mt/with-temp-file [file "temp-file"]
+    (testing (format "file = %s" (pr-str file))
+      (spit file "abc")
+      (is (u.files/regular-file? (u.files/get-path file)))))
+  (mt/with-temp-dir [dir "temp-dir"]
+    (testing (format "dir = %s" (pr-str dir)))
+    (let [file-in-dir (str (u.files/get-path dir "file"))]
+      (testing (format "file = %s" (pr-str file-in-dir))
+        (spit file-in-dir "abc") ; create a file in the dir to make sure it exists
+        (is (u.files/regular-file? (u.files/get-path file-in-dir)))))
+    (is (not (u.files/regular-file? (u.files/get-path dir))))))