[perf] Improve DB syncing times (#46103)

* perf: Add explicit .entryAt method for SnakeHatingMap

Otherwise, SHM invokes .keySet on itself which triggers the contstruction of the list of keys on each map access.

* perf: Cache (un)successful requiring of EE in dynamic-ee-oss-fn

* perf: Add a quicker path for getting metadatas from cache

* perf: Minor improvements

src/metabase/lib/metadata/cached_provider.cljc

@@ -51,18 +51,19 @@
 (defn- metadatas [cache uncached-provider metadata-type ids]
   (when (seq ids)
     (log/tracef "Getting %s metadata with IDs %s" metadata-type (pr-str (sort ids)))
-    (let [existing-ids (set (keys (get @cache metadata-type)))
-          missing-ids  (set/difference (set ids) existing-ids)]
-      (log/tracef "Already fetched %s: %s" metadata-type (pr-str (sort (set/intersection (set ids) existing-ids))))
-      (when (seq missing-ids)
-        (log/tracef "Need to fetch %s: %s" metadata-type (pr-str (sort missing-ids)))
-        ;; TODO -- we should probably store `::nil` markers for things we tried to fetch that didn't exist
-        (doseq [instance (lib.metadata.protocols/metadatas uncached-provider metadata-type missing-ids)]
-          (store-in-cache! cache [metadata-type (:id instance)] instance))))
+    (let [metadata-cache (get @cache metadata-type)]
+      (when-not (every? #(contains? metadata-cache %) ids)
+        (let [existing-ids (set (keys metadata-cache))
+              missing-ids  (set/difference (set ids) existing-ids)]
+          (log/tracef "Already fetched %s: %s" metadata-type (pr-str (sort (set/intersection (set ids) existing-ids))))
+          (when (seq missing-ids)
+            (log/tracef "Need to fetch %s: %s" metadata-type (pr-str (sort missing-ids)))
+            ;; TODO -- we should probably store `::nil` markers for things we tried to fetch that didn't exist
+            (doseq [instance (lib.metadata.protocols/metadatas uncached-provider metadata-type missing-ids)]
+              (store-in-cache! cache [metadata-type (:id instance)] instance))))))
     (into []
-          (comp (map (fn [id]
-                       (get-in-cache cache [metadata-type id])))
-                (filter some?))
+          (keep (fn [id]
+                  (get-in-cache cache [metadata-type id])))
           ids)))
 
 (defn- cached-metadatas [cache metadata-type metadata-ids]

src/metabase/lib/util/match/impl.cljc

@@ -41,8 +41,9 @@
   [replace-fn clause-parents form]
   (cond
     (map? form)
-    (into form (for [[k v] form]
-                 [k (replace-fn (conj clause-parents k) v)]))
+    (reduce-kv (fn [form k v]
+                 (assoc form k (replace-fn (conj clause-parents k) v)))
+               form form)
 
     (sequential? form)
     (mapv (partial replace-fn (if (keyword? (first form))

src/metabase/public_settings/premium_features.clj

@@ -549,18 +549,19 @@
   availability of EE code and the necessary premium feature. Returns a fn which, when invoked, applies its args to one
   of the EE implementation, the OSS implementation, or the fallback function."
   [ee-ns ee-fn-name]
-  (fn [& args]
-    (u/ignore-exceptions (classloader/require ee-ns))
-    (let [{:keys [ee oss feature fallback]} (get @registry ee-fn-name)]
-      (cond
-        (and ee (check-feature feature))
-        (apply ee args)
-
-        (and ee (fn? fallback))
-        (apply fallback args)
-
-        :else
-        (apply oss args)))))
+  (let [try-require-ee-ns-once (delay (u/ignore-exceptions (classloader/require ee-ns)))]
+    (fn [& args]
+      @try-require-ee-ns-once
+      (let [{:keys [ee oss feature fallback]} (get @registry ee-fn-name)]
+        (cond
+          (and ee (check-feature feature))
+          (apply ee args)
+
+          (and ee (fn? fallback))
+          (apply fallback args)
+
+          :else
+          (apply oss args))))))
 
 (defn- validate-ee-args
   "Throws an exception if the required :feature option is not present."

src/metabase/sync/sync_metadata/fields/common.clj

@@ -63,9 +63,10 @@
   corresponding Metabase Field for field metadata from the DB, or vice versa. Will prefer exact matches."
   [field-metadata :- TableMetadataFieldWithOptionalID
    other-metadata :- [:set TableMetadataFieldWithOptionalID]]
-  (let [matches (keep
+  (let [field-meta-canonical (canonical-name field-metadata)
+        matches (keep
                   (fn [other-field-metadata]
-                    (when (= (canonical-name field-metadata)
+                    (when (= field-meta-canonical
                              (canonical-name other-field-metadata))
                       other-field-metadata))
                   other-metadata)]

src/metabase/util/snake_hating_map.clj

@@ -12,6 +12,8 @@
    [potemkin :as p]
    [pretty.core :as pretty]))
 
+(set! *warn-on-reflection* true)
+
 (defn- snake-cased-key? [k]
   (some-> k (str/includes? "_")))
 
@@ -50,6 +52,9 @@
     (keys m))
   (meta [_this]
     (meta m))
+  (entryAt [this k]
+    (when (contains? m k)
+      (potemkin.PersistentMapProxy$MapEntry. this k)))
   (with-meta [this metta]
     (let [m' (with-meta m metta)]
       (if (identical? m m')