Limit model options to only GPT (#29889)

Limiting viewable LLM modes to the latest version of each generation of GPT.

src/metabase/metabot/settings.clj

@@ -2,6 +2,7 @@
   (:require
    [clojure.core.memoize :as memoize]
    [metabase.models.setting :refer [defsetting]]
+   [metabase.util :as u]
    [metabase.util.i18n :refer [deferred-tru]]
    [metabase.util.log :as log]
    [wkok.openai-clojure.api :as openai.api]))
@@ -41,6 +42,33 @@
   :visibility :internal
   :default 3)
 
+(defn- select-models
+  "Downselect the available openai models to only the latest version of each GPT family."
+  [models]
+  (->> models
+       (map (fn [{:keys [id] :as m}]
+              (when-some [[_ v r] (re-matches #"gpt-([\d\.]+)(.*)"
+                                              (u/lower-case-en id))]
+                (let [version (parse-double v)]
+                  (assoc m
+                    :version version
+                    :version-string v
+                    :generation (int version)
+                    :details r)))))
+       ;; Drop anything that doesn't match
+       (filter identity)
+       ;; Order by generation (asc), version (desc),
+       ;; length of details string (asc), length of version string (desc)
+       (sort-by (juxt :generation
+                      (comp - :version)
+                      (comp count :details)
+                      (comp - count :version-string)))
+       ;; Split out each generation
+       (partition-by :generation)
+       ;; Take the top item in each partition and select what we want
+       (map (comp #(select-keys % [:id :owned_by]) first))
+       reverse))
+
 (def ^:private memoized-fetch-openai-models
   (memoize/ttl
    ^{::memoize/args-fn (fn [[api-key organization]] [api-key organization])}
@@ -50,8 +78,7 @@
              {:api-key      api-key
               :organization organization})
             :data
-            (map #(select-keys % [:id :owned_by]))
-            (sort-by :id))
+            select-models)
        (catch Exception _
          (log/warn "Unable to fetch openai models.")
          [])))

test/metabase/metabot/settings_test.clj

+(ns metabase.metabot.settings-test
+  (:require
+   [clojure.test :refer :all]
+   [metabase.metabot.settings :as metabot-settings]))
+
+(deftest select-models-test
+  (testing "Ensure we are downselecting to the right models."
+    (let [wierd-models (shuffle
+                        [{:id "froob", :owned_by "openai"}
+                         {:id "froob-32", :owned_by "openai"}
+                         {:id "gpt-3.5-turbo", :owned_by "openai"}
+                         {:id "gpt-3.5-turbo-0301", :owned_by "openai"}
+                         {:id "gpt-4", :owned_by "openai"}
+                         {:id "GPT-4.1", :owned_by "openai"}
+                         {:id "gpt-4.0", :owned_by "openai"}
+                         {:id "gpt-4.55", :owned_by "openai"}
+                         {:id "gpt-4-0314", :owned_by "openai"}
+                         {:id "gpt-5", :owned_by "openai"}
+                         {:id "gpt-5.0", :owned_by "openai"}])]
+      (is (= [{:id "gpt-5.0", :owned_by "openai"}
+              {:id "gpt-4.55", :owned_by "openai"}
+              {:id "gpt-3.5-turbo", :owned_by "openai"}]
+             (#'metabot-settings/select-models wierd-models))))
+    (let [realistic-models (shuffle
+                            [{:id "gpt-3.5-turbo", :owned_by "openai"}
+                             {:id "gpt-3.5-turbo-0301", :owned_by "openai"}
+                             {:id "gpt-4", :owned_by "openai"}
+                             {:id "gpt-4-0314", :owned_by "openai"}])]
+      (is (= [{:id "gpt-4", :owned_by "openai"}
+              {:id "gpt-3.5-turbo", :owned_by "openai"}]
+             (#'metabot-settings/select-models realistic-models)))))
+  (testing "Select 'baseline' versions over checkpoints."
+    (let [version-string-case (shuffle
+                               [{:id "gpt-4", :owned_by "openai"}
+                                {:id "gpt-4-0101", :owned_by "openai"}])]
+      (is (= [{:id "gpt-4", :owned_by "openai"}]
+             (#'metabot-settings/select-models version-string-case)))))
+  (testing "Choose higher numbered versions over lower ones."
+    (let [version-string-case (shuffle
+                               [{:id "gpt-4", :owned_by "openai"}
+                                {:id "gpt-4.2", :owned_by "openai"}])]
+      (is (= [{:id "gpt-4.2", :owned_by "openai"}]
+             (#'metabot-settings/select-models version-string-case)))))
+  (testing "All other things being equal, choose the one with the longer version number."
+    (let [version-string-case (shuffle
+                               [{:id "gpt-5", :owned_by "openai"}
+                                {:id "gpt-5.0", :owned_by "openai"}])]
+      (is (= [{:id "gpt-5.0", :owned_by "openai"}]
+             (#'metabot-settings/select-models version-string-case))))))