From bb3194ba5fbe4b507fdb8bb1bc57ba34635fdb9d Mon Sep 17 00:00:00 2001
From: Alexander Solovyov <alexander@solovyov.net>
Date: Tue, 23 Apr 2024 15:52:42 +0300
Subject: [PATCH] move cache business logic to model from api (#41677)
---
src/metabase/api/cache.clj | 113 ++++++---------------------
src/metabase/models/cache_config.clj | 93 ++++++++++++++++++++++
test/metabase/api/cache_test.clj | 5 +-
3 files changed, 121 insertions(+), 90 deletions(-)
diff --git a/src/metabase/api/cache.clj b/src/metabase/api/cache.clj
index eb8839ef4fb..ede39f7372e 100644
--- a/src/metabase/api/cache.clj
+++ b/src/metabase/api/cache.clj
@@ -2,16 +2,11 @@
(:require
[clojure.walk :as walk]
[compojure.core :refer [GET]]
- [java-time.api :as t]
[metabase.api.common :as api]
[metabase.api.common.validation :as validation]
- [metabase.db.query :as mdb.query]
- [metabase.events :as events]
- [metabase.models :refer [CacheConfig Card]]
[metabase.models.cache-config :as cache-config]
[metabase.public-settings.premium-features :as premium-features :refer [defenterprise]]
- [metabase.util :as u]
- [metabase.util.i18n :refer [tru]]
+ [metabase.util.i18n :refer [tru trun]]
[metabase.util.malli.schema :as ms]
[toucan2.core :as t2]))
@@ -49,7 +44,6 @@
[]
(drop-internal-fields (CacheStrategy)))
-(def ^:private CachingModel [:enum "root" "database" "dashboard" "question"])
(defn- assert-valid-models [model ids premium?]
(cond
@@ -68,98 +62,40 @@
"question" :model/Card)
:id [:in ids]))))
-(defn- audit-caching-change! [id prev new]
- (events/publish-event!
- :event/cache-config-update
- {:user-id api/*current-user-id*
- :model :model/CacheConfig
- :model-id id
- :details {:model (or (:model prev) (:model new))
- :model-id (or (:model_id prev) (:model_id new))
- :old-value (dissoc prev :model :model_id)
- :new-value (dissoc new :model :model_id)}}))
-
(api/defendpoint GET "/"
"Return cache configuration."
[:as {{:strs [model collection]
:or {model "root"}}
:query-params}]
- {model (ms/QueryVectorOf CachingModel)
+ {model (ms/QueryVectorOf cache-config/CachingModel)
;; note that `nil` in `collection` means all configurations not scoped to any particular collection
collection [:maybe ms/PositiveInt]}
(validation/check-has-application-permission :setting)
- (let [items (if (premium-features/enable-cache-granular-controls?)
- (t2/select :model/CacheConfig
- :model [:in model]
- {:left-join [:report_card [:and
- [:= :model [:inline "question"]]
- [:= :model_id :report_card.id]
- [:= :report_card.collection_id collection]]
- :report_dashboard [:and
- [:= :model [:inline "dashboard"]]
- [:= :model_id :report_dashboard.id]
- [:= :report_dashboard.collection_id collection]]]
- :where [:case
- [:= :model [:inline "question"]] [:!= :report_card.id nil]
- [:= :model [:inline "dashboard"]] [:!= :report_dashboard.id nil]
- :else true]})
- (t2/select :model/CacheConfig :model "root"))]
- {:data (mapv cache-config/row->config items)}))
+ (when (and (not (premium-features/enable-cache-granular-controls?))
+ (not= model ["root"]))
+ (throw (premium-features/ee-feature-error (tru "Granular Caching"))))
+ {:data (cache-config/get-list model collection)})
(api/defendpoint PUT "/"
"Store cache configuration."
[:as {{:keys [model model_id strategy] :as config} :body}]
- {model CachingModel
+ {model cache-config/CachingModel
model_id ms/IntGreaterThanOrEqualToZero
strategy (CacheStrategyAPI)}
(validation/check-has-application-permission :setting)
(assert-valid-models model [model_id] (premium-features/enable-cache-granular-controls?))
- (t2/with-transaction [_tx]
- (let [data (cache-config/config->row config)
- current (t2/select-one :model/CacheConfig :model model :model_id model_id {:for :update})]
- {:id (u/prog1 (mdb.query/update-or-insert! :model/CacheConfig {:model model :model_id model_id}
- (constantly data))
- (audit-caching-change! <> current data))})))
+ {:id (cache-config/store! api/*current-user-id* config)})
(api/defendpoint DELETE "/"
- "Delete cache configuration."
+ "Delete cache configurations."
[:as {{:keys [model model_id]} :body}]
- {model CachingModel
+ {model cache-config/CachingModel
model_id (ms/QueryVectorOf ms/IntGreaterThanOrEqualToZero)}
(validation/check-has-application-permission :setting)
(assert-valid-models model model_id (premium-features/enable-cache-granular-controls?))
- (when-let [current (seq (t2/select :model/CacheConfig :model model :model_id [:in model_id]))]
- (t2/delete! :model/CacheConfig :model model :model_id [:in model_id])
- (doseq [item current]
- (audit-caching-change! (:id item)
- (select-keys item [:strategy :config :model :model_id])
- nil)))
+ (cache-config/delete! api/*current-user-id* model model_id)
nil)
-(defn- invalidate-cache-configs [database dashboard question]
- (let [conditions (for [[k vs] [[:database database]
- [:dashboard dashboard]
- [:question question]]
- v vs]
- [:and [:= :model (name k)] [:= :model_id v]])]
- (if (empty? conditions)
- 0
- ;; using JVM date rather than DB time since it's what are used in cache tasks
- (t2/query-one {:update (t2/table-name CacheConfig)
- :set {:invalidated_at (t/offset-date-time)}
- :where (into [:or] conditions)}))))
-
-(defn- invalidate-cards [database dashboard question]
- (let [card-ids (concat
- question
- (when (seq database)
- (t2/select-fn-vec :id [:model/Card :id] :database_id [:in database]))
- (when (seq dashboard)
- (t2/select-fn-vec :card_id [:model/DashboardCard :card_id] :dashboard_id [:in dashboard])))]
- (if (empty? card-ids)
- 0
- (t2/update! Card :id [:in card-ids]
- {:cache_invalidated_at (t/offset-date-time)}))))
(api/defendpoint POST "/invalidate"
"Invalidate cache entries.
@@ -178,19 +114,18 @@
(when-not (premium-features/enable-cache-granular-controls?)
(throw (premium-features/ee-feature-error (tru "Granular Caching"))))
- (let [cnt (if (= include :overrides)
- (invalidate-cards database dashboard question)
- (invalidate-cache-configs database dashboard question))]
- (case [(= include :overrides) (pos? cnt)]
- [true false] {:status 404
- :body {:message (tru "Could not find any cached questions for the given database, dashboard, or questions ids.")}}
- [true true] {:status 200
- :body {:message (tru "Invalidated cache for {0} question(s)." cnt)
- :count cnt}}
- [false false] {:status 404
- :body {:message (tru "Could not find a cache configuration to invalidate.")}}
- [false true] {:status 200
- :body {:message (tru "Invalidated {0} cache configuration(s)." cnt)
- :count cnt}})))
+ (let [cnt (cache-config/invalidate! {:databases database
+ :dashboards dashboard
+ :questions question
+ :with-overrides? (= include :overrides)})]
+ {:status (if (= cnt -1) 404 200)
+ :body {:count cnt
+ :message (case [(= include :overrides) (if (pos? cnt) 1 cnt)]
+ [true -1] (tru "Could not find a question for the criteria you've specified.")
+ [true 0] (tru "No cached results to invalidate.")
+ [true 1] (trun "Invalidated a cached result." "Invalidated {0} cached results." cnt)
+ [false -1] (tru "Nothing to invalidate.")
+ [false 0] (tru "No cache configuration to invalidate.")
+ [false 1] (trun "Invalidated cache configuration." "Invalidated {0} cache configurations." cnt))}}))
(api/define-routes)
diff --git a/src/metabase/models/cache_config.clj b/src/metabase/models/cache_config.clj
index f7f5f450e47..10996885359 100644
--- a/src/metabase/models/cache_config.clj
+++ b/src/metabase/models/cache_config.clj
@@ -3,10 +3,15 @@
(:require
[java-time.api :as t]
[medley.core :as m]
+ [metabase.db.query :as mdb.query]
+ [metabase.events :as events]
[metabase.models.interface :as mi]
+ [metabase.util :as u]
[methodical.core :as methodical]
[toucan2.core :as t2]))
+(def CachingModel "Caching is configurable for those models" [:enum "root" "database" "dashboard" "question"])
+
(def CacheConfig "Cache configuration" :model/CacheConfig)
(doto :model/CacheConfig
@@ -20,6 +25,17 @@
:config mi/transform-json
:state mi/transform-json})
+(defn- audit-caching-change! [user-id id prev new]
+ (events/publish-event!
+ :event/cache-config-update
+ {:user-id user-id
+ :model :model/CacheConfig
+ :model-id id
+ :details {:model (or (:model prev) (:model new))
+ :model-id (or (:model_id prev) (:model_id new))
+ :old-value (dissoc prev :model :model_id)
+ :new-value (dissoc new :model :model_id)}}))
+
;;; API
(defn root-strategy
@@ -50,3 +66,80 @@
:model_id model_id
:strategy (:type strategy)
:config (dissoc strategy :type)})
+
+(defn get-list
+ "Get a list of cache configurations for given `models` and a `collection`."
+ [models collection]
+ (->> (t2/select
+ :model/CacheConfig
+ :model [:in models]
+ {:left-join [:report_card [:and
+ [:= :model [:inline "question"]]
+ [:= :model_id :report_card.id]
+ [:= :report_card.collection_id collection]]
+ :report_dashboard [:and
+ [:= :model [:inline "dashboard"]]
+ [:= :model_id :report_dashboard.id]
+ [:= :report_dashboard.collection_id collection]]]
+ :where [:case
+ [:= :model [:inline "question"]] [:!= :report_card.id nil]
+ [:= :model [:inline "dashboard"]] [:!= :report_dashboard.id nil]
+ :else true]})
+ (mapv row->config)))
+
+(defn store!
+ "Store cache configuration in DB."
+ [user-id {:keys [model model_id] :as config}]
+ (t2/with-transaction [_tx]
+ (let [data (config->row config)
+ current (t2/select-one :model/CacheConfig :model model :model_id model_id {:for :update})]
+ (u/prog1 (mdb.query/update-or-insert! :model/CacheConfig {:model model :model_id model_id}
+ (constantly data))
+ (audit-caching-change! user-id <> current data)))))
+
+(defn delete!
+ "Delete cache configuration (possibly multiple), identified by a `model` and a vector of `model-ids`."
+ [user-id model model-ids]
+ (when-let [current (seq (t2/select :model/CacheConfig :model model :model_id [:in model-ids]))]
+ (t2/delete! :model/CacheConfig :model model :model_id [:in model-ids])
+ (doseq [item current]
+ (audit-caching-change! user-id
+ (:id item)
+ (select-keys item [:strategy :config :model :model_id])
+ nil))))
+
+
+;;; Invalidation
+
+(defn- invalidate-cards [databases dashboards questions]
+ (let [card-ids (concat
+ questions
+ (when (seq databases)
+ (t2/select-fn-vec :id [:model/Card :id] :database_id [:in databases]))
+ (when (seq dashboards)
+ (t2/select-fn-vec :card_id [:model/DashboardCard :card_id] :dashboard_id [:in dashboards])))]
+ (if (empty? card-ids)
+ -1
+ (t2/update! :model/Card :id [:in card-ids]
+ {:cache_invalidated_at (t/offset-date-time)}))))
+
+(defn- invalidate-cache-configs [databases dashboards questions]
+ (let [conditions (for [[k vs] [[:database databases]
+ [:dashboard dashboards]
+ [:question questions]]
+ v vs]
+ [:and [:= :model (name k)] [:= :model_id v]])]
+ (if (empty? conditions)
+ -1
+ ;; using JVM date rather than DB time since it's what are used in cache tasks
+ (t2/query-one {:update (t2/table-name CacheConfig)
+ :set {:invalidated_at (t/offset-date-time)}
+ :where (into [:or] conditions)}))))
+
+(defn invalidate!
+ "Invalidate cache configuration. Accepts lists of ids for different types of models. If `with-overrides?` is passed,
+ then invalidates cache on each individual card suitable for criteria."
+ [{:keys [databases dashboards questions with-overrides?]}]
+ (if with-overrides?
+ (invalidate-cards databases dashboards questions)
+ (invalidate-cache-configs databases dashboards questions)))
diff --git a/test/metabase/api/cache_test.clj b/test/metabase/api/cache_test.clj
index 3cb14ca5c62..8837ba4e75a 100644
--- a/test/metabase/api/cache_test.clj
+++ b/test/metabase/api/cache_test.clj
@@ -15,7 +15,10 @@
(mt/user-http-request :crowberto :put 402 "cache/"
{:model "question"
:model_id 1
- :strategy {:type "nocache"}}))))
+ :strategy {:type "nocache"}})))
+ (is (= "Granular Caching is a paid feature not currently available to your instance. Please upgrade to use it. Learn more at metabase.com/upgrade/"
+ (mt/user-http-request :crowberto :get 402 "cache/"
+ :model "question"))))
(testing "Can operate on root settings though"
(is (mt/user-http-request :crowberto :put 200 "cache/"
{:model "root"
--
GitLab