Entity id translation service (#47094)

* entity id translation + tests

* add api level test

* simplify definition of eid-table->model + add test

* update tests to take keywords

* improve comment

* generate the eid-table->model map

* delete now-obsolete test

* make it work in oss

* put the resulting response into a key, so we can add more information later

* formatting

* use model names without the model/ prefix as keys

* Creates list of `api/model->db-model`

- update keys for util/entity_id request
- update shape of util/entity_id response
- add test for not-found eids

* formatting

* Respond to code review feedback

---------

Co-authored-by: Oisin Coveney <oisin@metabase.com>

src/metabase/api/common.clj

@@ -730,3 +730,27 @@
                    (map #(assoc % ::model model) (f model items))))
          (sort-by (comp id+model->order (juxt :id ::model)))
          (map #(dissoc % ::model)))))
+
+(def model->db-model
+  "A mapping from the name of a model used in the API to information about it. This is reused in search, and entity_id
+  translation."
+  {"action"            {:db-model :model/Action             :alias :action}
+   "card"              {:db-model :model/Card               :alias :card}
+   "collection"        {:db-model :model/Collection         :alias :collection}
+   "dashboard"         {:db-model :model/Dashboard          :alias :dashboard}
+   "database"          {:db-model :model/Database           :alias :database}
+   "dataset"           {:db-model :model/Card               :alias :card}
+   "indexed-entity"    {:db-model :model/ModelIndexValue    :alias :model-index-value}
+   "metric"            {:db-model :model/Card               :alias :card}
+   "segment"           {:db-model :model/Segment            :alias :segment}
+   "snippet"           {:db-model :model/NativeQuerySnippet :alias :snippet}
+   "table"             {:db-model :model/Table              :alias :table}
+   "dashboard-card"    {:db-model :model/DashboardCard      :alias :dashboard-card}
+   "dashboard-tab"     {:db-model :model/DashboardTab       :alias :dashboard-tab}
+   "dimension"         {:db-model :model/Dimension          :alias :dimension}
+   "permissions-group" {:db-model :model/PermissionsGroup   :alias :permissions-group}
+   "pulse"             {:db-model :model/Pulse              :alias :pulse}
+   "pulse-card"        {:db-model :model/PulseCard          :alias :pulse-card}
+   "pulse-channel"     {:db-model :model/PulseChannel       :alias :pulse-channel}
+   "timeline"          {:db-model :model/Timeline           :alias :timeline}
+   "user"              {:db-model :model/User               :alias :user}})

src/metabase/api/embed/common.clj

@@ -3,6 +3,8 @@
    [cheshire.core :as json]
    [clojure.set :as set]
    [clojure.string :as str]
+   [malli.core :as mc]
+   [malli.error :as me]
    [medley.core :as m]
    [metabase.api.card :as api.card]
    [metabase.api.common :as api]
@@ -488,3 +490,67 @@
                          e)]
           (log/errorf e "Chain filter error\n%s" (u/pprint-to-str (u/all-ex-data e)))
           (throw e))))))
+
+;;; -------------------------------------- Entity ID transformation functions ------------------------------------------
+
+(def ^:private api-models
+  "The models that we will service for entity-id transformations."
+  (->> (descendants :metabase/model)
+       (filter #(= (namespace %) "model"))
+       (filter (fn has-entity-id?
+                 [model] (or ;; toucan1 models
+                          (isa? model :metabase.models.interface/entity-id)
+                          ;; toucan2 models
+                          (isa? model :hook/entity-id))))
+       (map keyword)
+       set))
+
+(def ^:private api-name->model
+  "Map of model names used on the API to their corresponding model."
+  (->> api/model->db-model
+       (map (fn [[k v]] [(keyword k) (:db-model v)]))
+       (filter (fn [[_ v]] (contains? api-models v)))
+       (into {})))
+
+(def ^:private eid-api-models
+  "Sorted vec of api models that have an entity_id column"
+  (vec (sort (keys api-name->model))))
+
+(def ^:private ApiModel (into [:enum] eid-api-models))
+
+(def ^:private EntityId
+  "A Malli schema for an entity id, this is a little looser because it needs to be fast."
+  [:and {:description "entity_id"}
+   :string
+   [:fn {:error/fn (fn [{:keys [value]} _]
+                     (str "\"" value "\" should be 21 characters long, but it is " (count value)))}
+    (fn eid-length-good? [eid] (= 21 (count eid)))]])
+
+(def ^:private ModelToEntityIds
+  "A Malli schema for a map of model names to a sequence of entity ids."
+  (mc/schema [:map-of ApiModel [:sequential EntityId]]))
+
+(mu/defn- entity-ids->id-for-model
+  "Given a model and a sequence of entity ids on that model, return a pairs of entity-id, id."
+  [api-name eids]
+  (let [model (api-name->model api-name) ;; This lookup is safe because we've already validated the api-names
+        eid->id (into {} (t2/select-fn->fn :entity_id :id [model :id :entity_id] :entity_id [:in eids]))]
+    (mapv (fn [entity-id]
+            [entity-id (if-let [id (get eid->id entity-id)]
+                         {:id id :type api-name}
+                         {:type api-name :status "not-found"})])
+          eids)))
+
+(defn model->entity-ids->ids
+  "Given a map of model names to a sequence of entity-ids for each, return a map from entity-id -> id."
+  [model-key->entity-ids]
+  (when-not (mc/validate ModelToEntityIds model-key->entity-ids)
+    (throw (ex-info "Invalid format." {:explanation (me/humanize
+                                                     (me/with-spell-checking
+                                                       (mc/explain ModelToEntityIds model-key->entity-ids)))
+                                       :allowed-models (sort (keys api-name->model))
+                                       :status-code 400})))
+  (into {}
+        (mapcat
+         (fn [[model eids]] (entity-ids->id-for-model model eids))
+         model-key->entity-ids)))

src/metabase/api/util.clj

@@ -11,6 +11,7 @@
    [metabase.analytics.stats :as stats]
    [metabase.api.common :as api]
    [metabase.api.common.validation :as validation]
+   [metabase.api.embed.common :as api.embed.common]
    [metabase.config :as config]
    [metabase.logger :as logger]
    [metabase.public-settings.premium-features :as premium-features]
@@ -92,4 +93,10 @@
         headers   {"Content-Disposition" "attachment; filename=\"connection_pool_info.json\""}]
     (assoc (response/response {:connection-pools pool-info}) :headers headers, :status 200)))
 
+(api/defendpoint POST "/entity_id"
+  "Translate entity IDs to model IDs."
+  [:as {{:keys [entity_ids]} :body}]
+  {entity_ids :map}
+  {:entity_ids (api.embed.common/model->entity-ids->ids entity_ids)})
+
 (api/define-routes)

src/metabase/search/config.clj

@@ -3,6 +3,7 @@
    [cheshire.core :as json]
    [clojure.string :as str]
    [flatland.ordered.map :as ordered-map]
+   [metabase.api.common :as api]
    [metabase.models.setting :refer [defsetting]]
    [metabase.permissions.util :as perms.u]
    [metabase.public-settings :as public-settings]
@@ -44,18 +45,22 @@
   "Show this many words of context before/after matches in long search results"
   2)
 
+(def excluded-models
+  "Set of models that should not be included in search results."
+  #{"dashboard-card"
+    "dashboard-tab"
+    "dimension"
+    "permissions-group"
+    "pulse"
+    "pulse-card"
+    "pulse-channel"
+    "snippet"
+    "timeline"
+    "user"})
+
 (def model-to-db-model
   "Mapping from string model to the Toucan model backing it."
-  {"action"         {:db-model :model/Action :alias :action}
-   "card"           {:db-model :model/Card :alias :card}
-   "collection"     {:db-model :model/Collection :alias :collection}
-   "dashboard"      {:db-model :model/Dashboard :alias :dashboard}
-   "database"       {:db-model :model/Database :alias :database}
-   "dataset"        {:db-model :model/Card :alias :card}
-   "indexed-entity" {:db-model :model/ModelIndexValue :alias :model-index-value}
-   "metric"         {:db-model :model/Card :alias :card}
-   "segment"        {:db-model :model/Segment :alias :segment}
-   "table"          {:db-model :model/Table :alias :table}})
+  (apply dissoc api/model->db-model excluded-models))
 
 (def all-models
   "Set of all valid models to search for. "

test/metabase/api/embed_test.clj

@@ -1742,3 +1742,89 @@
             (mt/with-temporary-setting-values [synchronous-batch-updates true]
               (client/client :get 202 (dashcard-url dashcard))
               (is (not= original-last-viewed-at (t2/select-one-fn :last_viewed_at :model/Dashboard :id dashboard-id))))))))))
+
+(deftest entity-id-single-card-translations-test
+  (mt/with-temp
+    [:model/Card {id   :id eid   :entity_id} {}]
+    (is (= {eid   {:id id   :type :card}}
+           (api.embed.common/model->entity-ids->ids {:card [eid]})))))
+
+(deftest entity-id-card-translations-test
+  (mt/with-temp
+    [:model/Card {id   :id eid   :entity_id} {}
+     :model/Card {id-0 :id eid-0 :entity_id} {}
+     :model/Card {id-1 :id eid-1 :entity_id} {}
+     :model/Card {id-2 :id eid-2 :entity_id} {}
+     :model/Card {id-3 :id eid-3 :entity_id} {}
+     :model/Card {id-4 :id eid-4 :entity_id} {}
+     :model/Card {id-5 :id eid-5 :entity_id} {}]
+    (is (= {eid   {:id id   :type :card}
+            eid-0 {:id id-0 :type :card}
+            eid-1 {:id id-1 :type :card}
+            eid-2 {:id id-2 :type :card}
+            eid-3 {:id id-3 :type :card}
+            eid-4 {:id id-4 :type :card}
+            eid-5 {:id id-5 :type :card}}
+           (api.embed.common/model->entity-ids->ids {:card [eid eid-0 eid-1 eid-2 eid-3 eid-4 eid-5]})))))
+
+(deftest entity-id-mixed-translations-test
+  (mt/with-temp
+    [;; prereqs to create the eid-able entities:
+     :model/Card  {model-id :id} {:type :model}
+     :model/Card  {card-id :id}  {}
+     :model/Field {field-id :id} {}
+
+     ;; eid models:
+     :model/Action             {action_id               :id action_eid               :entity_id} {:name "model for creating action" :model_id model-id :type :http}
+     :model/Collection         {collection_id           :id collection_eid           :entity_id} {}
+     ;; filling entity id for User doesn't work: do it manually below.
+     :model/User               {core_user_id            :id #_#_core_user_eid        :entity_id} {}
+     :model/Dimension          {dimension_id            :id dimension_eid            :entity_id} {:field_id field-id}
+     :model/NativeQuerySnippet {native_query_snippet_id :id native_query_snippet_eid :entity_id} {:creator_id core_user_id}
+     :model/PermissionsGroup   {permissions_group_id    :id permissions_group_eid    :entity_id} {}
+     :model/Pulse              {pulse_id                :id pulse_eid                :entity_id} {}
+     :model/PulseCard          {pulse_card_id           :id pulse_card_eid           :entity_id} {:pulse_id pulse_id :card_id card-id}
+     :model/PulseChannel       {pulse_channel_id        :id pulse_channel_eid        :entity_id} {:pulse_id pulse_id}
+     :model/Card               {report_card_id          :id report_card_eid          :entity_id} {}
+     :model/Dashboard          {report_dashboard_id     :id report_dashboard_eid     :entity_id} {}
+     :model/DashboardTab       {dashboard_tab_id        :id dashboard_tab_eid        :entity_id} {:dashboard_id report_dashboard_id}
+     :model/DashboardCard      {report_dashboardcard_id :id report_dashboardcard_eid :entity_id} {:dashboard_id report_dashboard_id}
+     :model/Segment            {segment_id              :id segment_eid              :entity_id} {}
+     :model/Timeline           {timeline_id             :id timeline_eid             :entity_id} {}]
+    (let [core_user_eid (u/generate-nano-id)]
+      (t2/update! :model/User core_user_id {:entity_id core_user_eid})
+      (is (= {action_eid               {:id action_id :type :action}
+              collection_eid           {:id collection_id :type :collection}
+              core_user_eid            {:id core_user_id :type :user}
+              dashboard_tab_eid        {:id dashboard_tab_id :type :dashboard-tab}
+              dimension_eid            {:id dimension_id :type :dimension}
+              native_query_snippet_eid {:id native_query_snippet_id :type :snippet}
+              permissions_group_eid    {:id permissions_group_id :type :permissions-group}
+              pulse_eid                {:id pulse_id :type :pulse}
+              pulse_card_eid           {:id pulse_card_id :type :pulse-card}
+              pulse_channel_eid        {:id pulse_channel_id :type :pulse-channel}
+              report_card_eid          {:id report_card_id :type :card}
+              report_dashboard_eid     {:id report_dashboard_id :type :dashboard}
+              report_dashboardcard_eid {:id report_dashboardcard_id :type :dashboard-card}
+              segment_eid              {:id segment_id :type :segment}
+              timeline_eid             {:id timeline_id :type :timeline}}
+             (api.embed.common/model->entity-ids->ids
+              {:action            [action_eid]
+               :card              [report_card_eid]
+               :collection        [collection_eid]
+               :dashboard         [report_dashboard_eid]
+               :dashboard-card    [report_dashboardcard_eid]
+               :dashboard-tab     [dashboard_tab_eid]
+               :dimension         [dimension_eid]
+               :permissions-group [permissions_group_eid]
+               :pulse             [pulse_eid]
+               :pulse-card        [pulse_card_eid]
+               :pulse-channel     [pulse_channel_eid]
+               :segment           [segment_eid]
+               :snippet           [native_query_snippet_eid]
+               :timeline          [timeline_eid]
+               :user              [core_user_eid]}))))))
+
+(deftest missing-entity-translations-test
+  (is (= {"abcdefghijklmnopqrstu" {:type :card, :status "not-found"}}
+         (api.embed.common/model->entity-ids->ids {:card ["abcdefghijklmnopqrstu"]}))))

test/metabase/api/util_test.clj

@@ -3,6 +3,7 @@
   (:require
    [clojure.string :as str]
    [clojure.test :refer :all]
+   [metabase.api.embed.common :as api.embed.common]
    [metabase.api.util :as api.util]
    [metabase.test :as mt]
    [metabase.util.log :as log]))
@@ -80,3 +81,16 @@
                                :email    "happy_user@test.com"
                                :source   "Analytics Inc"})
         (is (true? (deref sent? 2000 ::timedout)))))))
+
+(deftest entity-id-translation-test
+  (mt/with-temp [:model/Card {card-id :id card-eid :entity_id} {}]
+    (is (= {card-eid {:id card-id :type "card"}}
+           (-> (mt/user-http-request :crowberto :post 200
+                                     "util/entity_id"
+                                     {:entity_ids {"card" [card-eid]}})
+               :entity_ids
+               (update-keys name))))
+
+    (testing "error message contains allowed models"
+      (is (= (set (map name (keys @#'api.embed.common/api-name->model)))
+             (set (:allowed-models (mt/user-http-request :crowberto :post 400 "util/entity_id" {:entity_ids {"Card" [card-eid]}}))))))))