load analytics only when changed using a checksum (#38280)

* load analytics only when changed using a checksum

- over audit deserialization files

* update docstrings

* fix testing wording

e2e/runner/cypress-runner-backend.js

@@ -39,6 +39,7 @@ const CypressBackend = {
         MB_JETTY_PORT: server.port,
         MB_ENABLE_TEST_ENDPOINTS: "true",
         MB_DANGEROUS_UNSAFE_ENABLE_TESTING_H2_CONNECTIONS_DO_NOT_ENABLE: "true",
+        MB_LAST_ANALYTICS_CHECKSUM: "-1",
       };
 
       /**

enterprise/backend/src/metabase_enterprise/audit_db.clj

@@ -186,24 +186,70 @@
   :audit      :never
   :doc        false)
 
+(def ^:constant SKIP_CHECKSUM_FLAG
+  "If `last-analytics-checksum` is set to this value, we will skip calculating checksums entirely and *always* reload the
+  analytics data."
+  -1)
+
+(defsetting last-analytics-checksum
+  "A place to save the analytics-checksum, to check between app startups. If set to -1, skips the checksum process
+  entirely to avoid calculating checksums in environments (e2e tests) where we don't care."
+  :type       :integer
+  :default    0
+  :visibility :internal
+  :audit      :never
+  :doc        false
+  :export?    false)
+
+(defn- should-skip-checksum? [last-checksum]
+  (= SKIP_CHECKSUM_FLAG last-checksum))
+
+(defn analytics-checksum
+  "Hashes the contents of all non-dir files in the `analytics-dir-resource`."
+  []
+  (->> (io/file analytics-dir-resource)
+       file-seq
+       (remove fs/directory?)
+       (pmap #(hash (slurp %)))
+       (reduce + 0)))
+
+(defn- should-load-audit?
+  "Should we load audit data?"
+  [load-analytics-content? last-checksum current-checksum]
+  (and load-analytics-content?
+       (or (should-skip-checksum? last-checksum)
+           (not= last-checksum current-checksum))))
+
+(defn- get-last-and-current-checksum
+  "Gets the previous and current checksum for the analytics directory, respecting the `-1` flag for skipping checksums entirely."
+  []
+  (let [last-checksum (last-analytics-checksum)]
+    (if (should-skip-checksum? last-checksum)
+      [SKIP_CHECKSUM_FLAG SKIP_CHECKSUM_FLAG]
+      [last-checksum (analytics-checksum)])))
+
 (defn- maybe-load-analytics-content!
   [audit-db]
-  (when (and analytics-dir-resource (load-analytics-content))
-    (ee.internal-user/ensure-internal-user-exists!)
-    (adjust-audit-db-to-source! audit-db)
-    (log/info "Loading Analytics Content...")
-    (ia-content->plugins (plugins/plugins-dir))
-    (log/info (str "Loading Analytics Content from: " (instance-analytics-plugin-dir (plugins/plugins-dir))))
-    ;; The EE token might not have :serialization enabled, but audit features should still be able to use it.
-    (let [report (log/with-no-logs
-                   (serialization.cmd/v2-load-internal! (str (instance-analytics-plugin-dir (plugins/plugins-dir)))
-                                                        {:backfill? false}
-                                                        :token-check? false))]
-      (if (not-empty (:errors report))
-        (log/info (str "Error Loading Analytics Content: " (pr-str report)))
-        (log/info (str "Loading Analytics Content Complete (" (count (:seen report)) ") entities loaded."))))
-    (when-let [audit-db (t2/select-one :model/Database :is_audit true)]
-      (adjust-audit-db-to-host! audit-db))))
+  (when analytics-dir-resource
+    ;; can't calculate checksum unless there is an analytics-dir resource
+    (let [[last-checksum current-checksum] (get-last-and-current-checksum)]
+      (when (should-load-audit? load-analytics-content last-checksum current-checksum)
+        (last-analytics-checksum! current-checksum)
+        (ee.internal-user/ensure-internal-user-exists!)
+        (adjust-audit-db-to-source! audit-db)
+        (log/info "Loading Analytics Content...")
+        (ia-content->plugins (plugins/plugins-dir))
+        (log/info (str "Loading Analytics Content from: " (instance-analytics-plugin-dir (plugins/plugins-dir))))
+        ;; The EE token might not have :serialization enabled, but audit features should still be able to use it.
+        (let [report (log/with-no-logs
+                       (serialization.cmd/v2-load-internal! (str (instance-analytics-plugin-dir (plugins/plugins-dir)))
+                                                            {:backfill? false}
+                                                            :token-check? false))]
+          (if (not-empty (:errors report))
+            (log/info (str "Error Loading Analytics Content: " (pr-str report)))
+            (log/info (str "Loading Analytics Content Complete (" (count (:seen report)) ") entities loaded."))))
+        (when-let [audit-db (t2/select-one :model/Database :is_audit true)]
+          (adjust-audit-db-to-host! audit-db))))))
 
 (defn- maybe-install-audit-db
   []

enterprise/backend/test/metabase_enterprise/audit_db_test.clj

@@ -35,6 +35,8 @@
   (mt/test-drivers #{:postgres :h2 :mysql}
     (testing "Audit DB content is not installed when it is not found"
       (t2/delete! :model/Database :is_audit true)
+      ;; reset checksum
+      (audit-db/last-analytics-checksum! 0)
       (with-redefs [audit-db/analytics-dir-resource nil]
         (is (nil? @#'audit-db/analytics-dir-resource))
         (is (= ::audit-db/installed (audit-db/ensure-audit-db-installed!)))
@@ -42,7 +44,8 @@
             "Audit DB is installed.")
         (is (= 0 (t2/count :model/Card {:where [:= :database_id perms/audit-db-id]}))
             "No cards created for Audit DB."))
-      (t2/delete! :model/Database :is_audit true))
+      (t2/delete! :model/Database :is_audit true)
+      (audit-db/last-analytics-checksum! 0))
 
     (testing "Audit DB content is installed when it is found"
       (is (= ::audit-db/installed (audit-db/ensure-audit-db-installed!)))
@@ -129,3 +132,15 @@
         (testing "No exception is thrown when db has 'duplicate' entries."
           (is (= :metabase-enterprise.audit-db/no-op
                  (audit-db/ensure-audit-db-installed!))))))))
+
+(deftest should-load-audit?-test
+  (testing "load-analytics-content + checksums dont match => load"
+    (is (= (#'audit-db/should-load-audit? true 1 3) true)))
+  (testing "load-analytics-content + last-checksum is -1 => load (even if current-checksum is also -1)"
+    (is (= (#'audit-db/should-load-audit? true -1 -1) true)))
+  (testing "checksums are the same => do not load"
+    (is (= (#'audit-db/should-load-audit? true 3 3) false)))
+  (testing "load-analytics-content false => do not load"
+    (is (= (#'audit-db/should-load-audit? false 3 5) false)))
+  (testing "load-analytics-content is false + checksums do not match  => do not load"
+    (is (= (#'audit-db/should-load-audit? false 1 3) false))))