Analytics should load only when changed, via checksum (#38622)

* load analytics only when changed using a checksum

- over audit deserialization files

* update docstrings

* fix testing wording

* fix nil analytics-resource-dir

* in tests, when you delete the audit db, set the checksum back to 0

* add a comment

* WIP hashing ia content from inside a jar

* Do the unzip sooner, still hashes the file system

Co-authored-by: John Swanson <john.swanson@metabase.com>

* cleanup java-time -> java-time.api

---------

Co-authored-by: John Swanson <john.swanson@metabase.com>

dev/src/dev.clj

@@ -43,7 +43,7 @@
    [dev.model-tracking :as model-tracking]
    [hashp.core :as hashp]
    [honey.sql :as sql]
-   [java-time :as t]
+   [java-time.api :as t]
    [malli.dev :as malli-dev]
    [metabase.api.common :as api]
    [metabase.config :as config]

enterprise/backend/src/metabase_enterprise/audit_db.clj

@@ -18,7 +18,8 @@
    [metabase.util.log :as log]
    [toucan2.core :as t2])
   (:import
-   (java.util.jar JarEntry JarFile)))
+   (java.util.jar JarEntry JarFile)
+   (sun.nio.fs UnixPath)))
 
 (set! *warn-on-reflection* true)
 
@@ -157,6 +158,8 @@
   [plugins-dir]
   (fs/path (fs/absolutize plugins-dir) "instance_analytics"))
 
+(def ^:private jar-resource-path "instance_analytics/")
+
 (defn- ia-content->plugins
   "Load instance analytics content (collections/dashboards/cards/etc.) from resources dir or a zip file
    and copies it into the provided directory (by default, plugins/instance_analytics)."
@@ -167,7 +170,8 @@
     (if (running-from-jar?)
       (let [path-to-jar (get-jar-path)]
         (log/info "The app is running from a jar, starting copy...")
-        (copy-from-jar! path-to-jar "instance_analytics/" plugins-dir)
+        (log/info (str "Copying " path-to-jar "::" jar-resource-path " -> " plugins-dir))
+        (copy-from-jar! path-to-jar jar-resource-path plugins-dir)
         (log/info "Copying complete."))
       (let [in-path (fs/path analytics-dir-resource)]
         (log/info "The app is not running from a jar, starting copy...")
@@ -207,11 +211,12 @@
 (defn analytics-checksum
   "Hashes the contents of all non-dir files in the `analytics-dir-resource`."
   []
-  (->> (io/file analytics-dir-resource)
+  (->> ^UnixPath (instance-analytics-plugin-dir (plugins/plugins-dir))
+       (.toFile)
        file-seq
        (remove fs/directory?)
        (pmap #(hash (slurp %)))
-       (reduce + 0)))
+       (reduce +)))
 
 (defn- should-load-audit?
   "Should we load audit data?"
@@ -231,14 +236,12 @@
 (defn- maybe-load-analytics-content!
   [audit-db]
   (when analytics-dir-resource
-    ;; can't calculate checksum unless there is an analytics-dir resource
+    (ee.internal-user/ensure-internal-user-exists!)
+    (adjust-audit-db-to-source! audit-db)
+    (ia-content->plugins (plugins/plugins-dir))
     (let [[last-checksum current-checksum] (get-last-and-current-checksum)]
-      (when (should-load-audit? load-analytics-content last-checksum current-checksum)
+      (when (should-load-audit? (load-analytics-content) last-checksum current-checksum)
         (last-analytics-checksum! current-checksum)
-        (ee.internal-user/ensure-internal-user-exists!)
-        (adjust-audit-db-to-source! audit-db)
-        (log/info "Loading Analytics Content...")
-        (ia-content->plugins (plugins/plugins-dir))
         (log/info (str "Loading Analytics Content from: " (instance-analytics-plugin-dir (plugins/plugins-dir))))
         ;; The EE token might not have :serialization enabled, but audit features should still be able to use it.
         (let [report (log/with-no-logs
@@ -247,9 +250,9 @@
                                                             :token-check? false))]
           (if (not-empty (:errors report))
             (log/info (str "Error Loading Analytics Content: " (pr-str report)))
-            (log/info (str "Loading Analytics Content Complete (" (count (:seen report)) ") entities loaded."))))
-        (when-let [audit-db (t2/select-one :model/Database :is_audit true)]
-          (adjust-audit-db-to-host! audit-db))))))
+            (log/info (str "Loading Analytics Content Complete (" (count (:seen report)) ") entities loaded."))))))
+    (when-let [audit-db (t2/select-one :model/Database :is_audit true)]
+      (adjust-audit-db-to-host! audit-db))))
 
 (defn- maybe-install-audit-db
   []

enterprise/backend/test/metabase_enterprise/sandbox/models/params/field_values_test.clj

 (ns metabase-enterprise.sandbox.models.params.field-values-test
   (:require
    [clojure.test :refer :all]
-   [java-time :as t]
+   [java-time.api :as t]
    [metabase-enterprise.sandbox.models.group-table-access-policy
     :refer [GroupTableAccessPolicy]]
    [metabase-enterprise.sandbox.models.params.field-values

src/metabase/automagic_dashboards/interesting.clj

@@ -43,7 +43,7 @@
     [clojure.math.combinatorics :as math.combo]
     [clojure.string :as str]
     [clojure.walk :as walk]
-    [java-time :as t]
+    [java-time.api :as t]
     [medley.core :as m]
     [metabase.automagic-dashboards.dashboard-templates :as dashboard-templates]
     [metabase.automagic-dashboards.schema :as ads]

src/metabase/models/recent_views.clj

@@ -3,7 +3,7 @@
   each user."
   (:require
     #_{:clj-kondo/ignore [:deprecated-namespace]}
-   [java-time :as t]
+   [java-time.api :as t]
    [metabase.util :as u]
    [metabase.util.honey-sql-2 :as h2x]
    [metabase.util.malli :as mu]

test/metabase/actions/test_util.clj

@@ -2,7 +2,7 @@
   (:require
    [clojure.java.jdbc :as jdbc]
    [clojure.test :refer :all]
-   [java-time :as t]
+   [java-time.api :as t]
    [metabase.driver :as driver]
    [metabase.driver.ddl.interface :as ddl.i]
    [metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]

test/metabase/api/search_test.clj

@@ -3,7 +3,7 @@
    [clojure.set :as set]
    [clojure.string :as str]
    [clojure.test :refer :all]
-   [java-time :as t]
+   [java-time.api :as t]
    [metabase.analytics.snowplow-test :as snowplow-test]
    [metabase.api.search :as api.search]
    [metabase.mbql.normalize :as mbql.normalize]

test/metabase/models/recent_views_test.clj

 (ns metabase.models.recent-views-test
   (:require
    [clojure.test :refer :all]
-   [java-time :as t]
+   [java-time.api :as t]
    [metabase.models.recent-views :as recent-views]
    [metabase.test :as mt]
    [toucan2.core :as t2]