db changes tracking utilities (#28256)

dev/src/dev.clj

@@ -2,6 +2,7 @@
   "Put everything needed for REPL development within easy reach"
   (:require
    [clojure.core.async :as a]
+   [dev.model-tracking :as model-tracking]
    [dev.debug-qp :as debug-qp]
    [honeysql.core :as hsql]
    [malli.dev :as malli-dev]
@@ -29,13 +30,23 @@
    [toucan2.connection :as t2.connection]
    [toucan2.pipeline :as t2.pipeline]))
 
-(comment debug-qp/keep-me)
+(set! *warn-on-reflection* true)
+
+(comment
+  debug-qp/keep-me
+  model-tracking/keep-me)
 
 (defn tap>-spy [x]
   (doto x tap>))
 
 (p/import-vars
- [debug-qp process-query-debug])
+ [debug-qp process-query-debug]
+ [model-tracking
+  track!
+  untrack!
+  untrack-all!
+  reset-changes!
+  changes])
 
 (def initialized?
   (atom nil))

dev/src/dev/model_tracking.clj

+(ns dev.model-tracking
+  "A set of utility function to track model changes.
+  Use this when you want to observe changes of database models when doing stuffs on UI.
+
+  How to use this?
+    > (track! models/Dashboard models/Card models/DashboardCard)
+    -- Go on UI and do stuffs like (i.e: update viz-settings of a dashcard).
+
+    > (changes)
+    ;; => {:report_card {:insert ...}}
+
+  You can use [[reset-changes!]] to clear our all the current trackings.
+  And [[untrack-all!]] or [[untrack!]] to stop tracking."
+  (:require
+   [clojure.pprint :as pprint]
+   [metabase.util :as u]
+   [methodical.core :as m]
+   [toucan2.core :as t2]
+   [toucan2.model :as t2.model]
+   [toucan2.tools.before-delete :as t2.before-delete]
+   [toucan2.tools.before-insert :as t2.before-insert]
+   [toucan2.tools.before-update :as t2.before-update]
+   [toucan2.util :as t2.util]))
+
+
+(def changes*
+  "An atom to store all the changes of models that we currently track."
+  (atom {}))
+
+(def ^:private tracked-models (atom #{}))
+
+(defn on-change
+  "When a change occurred, execute this function.
+
+  Currently it just prints the console out to the console.
+  But if you prefer other method of debugging (i.e: tap), you can redef this function
+
+    (alter-var-root #'model-tracking/on-change (fn [path change] (tap> [path change])))
+
+
+  - path: is a element vector [model, action]
+  - change-info: is a map of the change for a model
+  "
+  [path change-info]
+  (println (u/colorize :magenta :new-change) (u/colorize :magenta path))
+  (pprint/pprint change-info))
+
+(defn- clean-change
+  [change]
+  (dissoc change :updated_at :created_at))
+
+(defn- new-change
+  "Add a change to the [[changes]] atom.
+
+    > (new-change :models/Card :insert {:name \"new card\"})
+    instance
+
+    > @changes*
+    {:report_card {:insert [{:name \"new card\"}]}]}.
+
+  For insert, track the instance as a map.
+  For update, only track the changes."
+  [model action row-or-instance]
+  (let [model       (t2/resolve-model model)
+        change-info (->> (case action
+                           :update
+                           (into {} (t2/changes row-or-instance))
+                           (into {} row-or-instance))
+                        clean-change)
+        path       [(t2/table-name model) action]]
+    ;; ideally this should be debug, but for some reasons this doesn't get logged
+    (on-change path change-info)
+    (swap! changes* update-in path concat [change-info])))
+
+(defn- new-change-thunk
+  [model action]
+  (fn [_model row]
+    (new-change model action row)
+    row))
+
+(def ^:private hook+aux-method+action+deriveable
+  "A list of toucan hooks that we will subscribed to when tracking a model."
+  [;; will be better if we could use after-insert to get the inserted id, but toucan2 doesn't define a multimethod for after-insert
+   [#'t2.before-insert/before-insert :after :insert ::t2.before-insert/before-insert]
+   [#'t2.before-update/before-update :after :update ::t2.before-update/before-update]
+   ;; we do :before aux-method instead of :after for delete bacause the after method has input is number of affected rows
+   [#'t2.before-delete/before-delete :before :delete ::t2.before-delete/before-delete]])
+
+(defn- track-one!
+  [model]
+  (doseq [[hook aux-method action deriveable] hook+aux-method+action+deriveable]
+    (when-not (m/primary-method @hook model)
+      ;; aux-method will not be triggered if there isn't a primary method
+      (t2.util/maybe-derive model deriveable)
+      (m/add-primary-method! hook model (fn [_ _model row] row)))
+    (m/add-aux-method-with-unique-key! hook aux-method model (new-change-thunk model action) ::tracking)))
+
+(defn track!
+  "Start tracking a list of models.
+
+  (track! 'Card 'Dashboard)"
+  [& models]
+  (doseq [model (map t2.model/resolve-model models)]
+    (track-one! model)
+    (swap! tracked-models conj model)))
+
+(defn- untrack-one!
+  [model]
+  (doseq [[hook aux-method _action] hook+aux-method+action+deriveable]
+    (m/remove-aux-method-with-unique-key! hook aux-method model ::tracking)
+    (swap! tracked-models disj model)))
+
+(defn untrack!
+  "Remove tracking for a list of models.
+
+  (untrack! 'Card 'Dashboard)"
+  [& models]
+  (doseq [model (map t2.model/resolve-model models)]
+    (untrack-one! model)))
+
+(defn reset-changes!
+  "Empty all the recorded changes."
+  []
+  (reset! changes* {}))
+
+(defn untrack-all!
+  "Quickly untrack all the tracked models."
+  []
+  (reset-changes!)
+  (apply untrack! @tracked-models)
+  (reset! tracked-models #{}))
+
+(defn changes
+  "Return all changes that were recorded."
+  []
+  @changes*)

dev/test/dev/model_tracking_test.clj

+(ns dev.model-tracking-test
+  (:require
+   [clojure.test :refer :all]
+   [dev.model-tracking :as model-tracking]
+   [metabase.models :refer [Collection]]
+   [metabase.test :as mt]
+   [toucan2.core :as t2]))
+
+(use-fixtures :each (fn [thunk]
+                      (model-tracking/untrack-all!)
+                      (thunk)))
+
+(deftest e2e-test
+  (mt/with-model-cleanup [Collection]
+    ;; setup
+    (model-tracking/track! 'Collection)
+
+    (testing "insert"
+      (t2/insert! Collection {:name "Test tracking" :color "#000000"})
+      (testing "should be tracked"
+        (is (=? [{:name  "Test tracking"
+                  :color "#000000"}]
+                (get-in (model-tracking/changes) [:collection :insert]))))
+      (testing "should take affects"
+        (is (= 1 (t2/count Collection :name "Test tracking")))))
+
+    (testing "update"
+      (t2/update! Collection {:name "Test tracking"} {:color "#ffffff"})
+      (testing "changes should be tracked"
+        (is (= [{:color "#ffffff"}]
+               (get-in (model-tracking/changes) [:collection :update]))))
+      (testing "should take affects"
+        (is (= "#ffffff" (t2/select-one-fn :color Collection :name "Test tracking")))))
+
+    (testing "delete"
+      (let [coll-id (t2/select-one-pk Collection :name "Test tracking")]
+        (t2/delete! Collection coll-id)
+        (testing "should be tracked"
+          (is (=? [{:color "#ffffff"
+                    :name  "Test tracking",
+                    :id    coll-id}]
+                  (get-in (model-tracking/changes) [:collection :delete]))))
+        (testing "should take affects"
+          (is (nil? (t2/select-one Collection :id coll-id))))))
+
+    (testing "untrack should stop all tracking for"
+      (model-tracking/untrack-all!)
+      (testing "insert"
+        (t2/insert! Collection {:name "Test tracking" :color "#000000"})
+        (testing "changes not should be tracked"
+          (is (empty? (model-tracking/changes))))
+        (testing "should take affects"
+          (is (= 1 (t2/count Collection :name "Test tracking")))))
+
+      (testing "update"
+        (t2/update! Collection {:name "Test tracking"} {:color "#ffffff"})
+        (testing "changes not should be tracked"
+          (is (empty? (model-tracking/changes))))
+        (testing "should take affects"
+          (is (= "#ffffff" (t2/select-one-fn :color Collection :name "Test tracking")))))
+
+      (testing "delete"
+        (let [coll-id (t2/select-one-pk Collection :name "Test tracking")]
+          (t2/delete! Collection coll-id)
+          (testing "changes not should be tracked"
+            (is (empty? (model-tracking/changes))))
+          (testing "should take affects"
+            (is (nil? (t2/select-one Collection :id coll-id)))))))))