Backport "fix hanging cache refresh jobs" (#39664) (#40539)

I started a much more complex fix with a job that tried to clean up
stale `refreshing` PersistedInfos... then talked to Dan, who pointed out
that because only one instance is running these jobs at any given
time, we can just add `refreshing` to the list of refreshable states.

enterprise/backend/src/metabase_enterprise/advanced_config/caching.clj

@@ -16,7 +16,7 @@
   "States of `persisted_info` records which can be refreshed."
   :feature :cache-granular-controls
   []
-  #{"creating" "persisted" "error"})
+  #{"refreshing" "creating" "persisted" "error"})
 
 (defenterprise prunable-states
   "States of `persisted_info` records which can be pruned."

src/metabase/models/persisted_info.clj

@@ -85,7 +85,7 @@
    refreshable with the feature flag disabled."
   metabase-enterprise.advanced-config.caching
   []
-  #{"creating" "persisted" "error" "off"})
+  #{"refreshing" "creating" "persisted" "error" "off"})
 
 (defenterprise prunable-states
   "States of `persisted_info` records which can be pruned."

test/metabase/task/persist_refresh_test.clj

@@ -114,6 +114,29 @@
                                  :key (format "metabase.task.PersistenceRefresh.database.trigger.%d" (u/the-id db-2))}}
                (job-info db-1 db-2)))))))
 
+(deftest fault-tolerance-test
+  (mt/with-model-cleanup [TaskHistory]
+    (mt/with-temp [Database db {:settings {:persist-models-enabled true}}
+                   Card model {:type "model" :database_id (u/the-id db)}
+                   PersistedInfo persisted-info {:card_id (u/the-id model) :database_id (u/the-id db)}]
+      (let [test-refresher (reify task.persist-refresh/Refresher
+                             (refresh! [_ _database _definition _card]
+                               {:state :success})
+                             (unpersist! [_ _database _persisted-info]))
+            original-update! t2/update!]
+        (testing "If saving the `persisted` (or `error`) state fails..."
+          (with-redefs [t2/update! (fn [model id update]
+                                     (when (= "persisted" (:state update))
+                                       (throw (ex-info "simulated error" {})))
+                                     (original-update! model id update))]
+            (is (thrown-with-msg? clojure.lang.ExceptionInfo #"simulated error"
+                                  (#'task.persist-refresh/refresh-tables! (u/the-id db) test-refresher))))
+          (testing "the PersistedInfo is left in the `refreshing` state"
+            (is (= "refreshing" (t2/select-one-fn :state :model/PersistedInfo :id (u/the-id persisted-info)))))
+          (testing "but a subsequent refresh run will refresh the table"
+            (#'task.persist-refresh/refresh-tables! (u/the-id db) test-refresher)
+            (is (= "persisted" (t2/select-one-fn :state :model/PersistedInfo :id (u/the-id persisted-info))))))))))
+
 (deftest refresh-tables!'-test
   (mt/with-model-cleanup [TaskHistory]
     (mt/with-temp [Database db {:settings {:persist-models-enabled true}}