Skip fingerprinting or scanning field values for fields with base-type=type/Collection (#45160)

src/metabase/db/query.clj

@@ -45,7 +45,7 @@
 (def ^:private NamespacedKeyword
   [:and :keyword [:fn (comp seq namespace)]])
 
-(mu/defn ^:private type-keyword->descendants :- [:set {:min 1} ms/NonBlankString]
+(mu/defn type-keyword->descendants :- [:set {:min 1} ms/NonBlankString]
   "Return a set of descendents of Metabase `type-keyword`. This includes `type-keyword` itself, so the set will always
   have at least one element.
 

src/metabase/models/field_values.clj

@@ -232,9 +232,11 @@
            has-field-values :has_field_values} field-or-field-id]
       (boolean
        (and
-        (not (isa? base-type :type/field-values-unsupported))
         (not (contains? #{:retired :sensitive :hidden :details-only} (keyword visibility-type)))
-        (not (isa? (keyword base-type) :type/Temporal))
+        (not (some #(isa? (keyword base-type) %) [:type/field-values-unsupported
+                                                  :type/Structured
+                                                  :type/Temporal
+                                                  :type/Collection]))
         (not (= (keyword base-type) :type/*))
         (#{:list :auto-list} (keyword has-field-values)))))))
 

src/metabase/sync/analyze/fingerprint.clj

@@ -160,8 +160,8 @@
     [:not (mdb.query/isa :semantic_type :type/PK)]
     [:= :semantic_type nil]]
    [:not-in :visibility_type ["retired" "sensitive"]]
-   [:not= :base_type (u/qualified-name :type/*)]
-   [:not (mdb.query/isa :base_type :type/Structured)]])
+   [:not-in :base_type (-> (set (mapcat mdb.query/type-keyword->descendants [:type/Structured :type/Collection]))
+                           (conj (u/qualified-name :type/*)))]])
 
 (def ^:dynamic *refingerprint?*
   "Whether we are refingerprinting or doing the normal fingerprinting. Refingerprinting should get fields that already

test/metabase/models/field_values_test.clj

@@ -96,7 +96,23 @@
                                    {{:has_field_values :list
                                      :visibility_type  :normal
                                      :base_type        :type/*}
-                                    false}}
+                                    false}
+
+                                   "type/Structured fields should be excluded"
+                                   (into {}
+                                         (for [base-type (conj (descendants :type/Structured) :type/Structured)]
+                                           [{:has_field_values :list
+                                             :visibility_type  :normal
+                                             :base_type        base-type}
+                                            false]))
+
+                                   "type/Collection fields should be excluded"
+                                   (into {}
+                                         (for [base-type (conj (descendants :type/Collection) :type/Collection)]
+                                           [{:has_field_values :list
+                                             :visibility_type  :normal
+                                             :base_type        base-type}
+                                            false]))}
           [input expected] input->expected]
     (testing (str group "\n")
       (testing (pr-str (list 'field-should-have-field-values? input))

test/metabase/sync/analyze/fingerprint_test.clj

@@ -26,6 +26,17 @@
   (is (= #{"type/ImageURL" "type/AvatarURL"}
          (#'sync.fingerprint/base-types->descendants #{:type/ImageURL :type/AvatarURL}))))
 
+(def ^:private skip-fingerprint-base-types
+  #{"type/*"
+    "type/Structured"
+    "type/SerializedJSON"
+    "type/JSON"
+    "type/Dictionary"
+    "type/Array"
+    "type/Collection"
+    "type/XML"
+    "type/DruidJSON"})
+
 (deftest ^:parallel honeysql-for-fields-that-need-fingerprint-updating-test
   (testing (str "Make sure we generate the correct HoneySQL WHERE clause based on whatever is in "
                 "`*fingerprint-version->types-that-should-be-re-fingerprinted*`")
@@ -36,8 +47,7 @@
               [:not (mdb.query/isa :semantic_type :type/PK)]
               [:= :semantic_type nil]]
              [:not-in :visibility_type ["retired" "sensitive"]]
-             [:not= :base_type "type/*"]
-             [:not (mdb.query/isa :base_type :type/Structured)]
+             [:not-in :base_type skip-fingerprint-base-types]
              [:or
               [:and
                [:< :fingerprint_version 1]
@@ -53,8 +63,7 @@
             [:not (mdb.query/isa :semantic_type :type/PK)]
             [:= :semantic_type nil]]
            [:not-in :visibility_type ["retired" "sensitive"]]
-           [:not= :base_type "type/*"]
-           [:not (mdb.query/isa :base_type :type/Structured)]
+           [:not-in :base_type skip-fingerprint-base-types]
            [:or
             [:and
              [:< :fingerprint_version 2]
@@ -76,8 +85,7 @@
               [:not (mdb.query/isa :semantic_type :type/PK)]
               [:= :semantic_type nil]]
              [:not-in :visibility_type ["retired" "sensitive"]]
-             [:not= :base_type "type/*"]
-             [:not (mdb.query/isa :base_type :type/Structured)]
+             [:not-in :base_type skip-fingerprint-base-types]
              [:or
               [:and
                [:< :fingerprint_version 2]
@@ -100,8 +108,7 @@
               [:not (mdb.query/isa :semantic_type :type/PK)]
               [:= :semantic_type nil]]
              [:not-in :visibility_type ["retired" "sensitive"]]
-             [:not= :base_type "type/*"]
-             [:not (mdb.query/isa :base_type :type/Structured)]
+             [:not-in :base_type skip-fingerprint-base-types]
              [:or
               [:and
                [:< :fingerprint_version 4]
@@ -129,8 +136,7 @@
                      [:not (mdb.query/isa :semantic_type :type/PK)]
                      [:= :semantic_type nil]]
                     [:not-in :visibility_type ["retired" "sensitive"]]
-                    [:not= :base_type "type/*"]
-                    [:not (mdb.query/isa :base_type :type/Structured)]]}
+                    [:not-in :base_type skip-fingerprint-base-types]]}
            (binding [sync.fingerprint/*refingerprint?* true]
              (#'sync.fingerprint/honeysql-for-fields-that-need-fingerprint-updating))))))