Add DashboardSubscription and Alert audit queries (#17818)

* Add DashboardSubscription and Alert table audit queries

* metastore -> premium-features

enterprise/backend/src/metabase_enterprise/audit_app/pages/alerts.clj

+(ns metabase-enterprise.audit-app.pages.alerts
+  (:require [clojure.string :as str]
+            [metabase-enterprise.audit-app.interface :as audit.i]
+            [metabase-enterprise.audit-app.pages.common :as common]
+            [metabase-enterprise.audit-app.pages.common.pulses :as common.pulses]))
+
+(def ^:private table-metadata
+  (into
+   [[:card_id   {:display_name "Question ID",  :base_type :type/Integer, :remapped_to :card_name}]
+    [:card_name {:display_name "Question Name" :base_type :type/Text,    :remapped_from :card_id}]]
+   common.pulses/table-metadata))
+
+(def ^:private table-query-columns
+  (into
+   [:card_id
+    :card_name]
+   common.pulses/table-query-columns))
+
+(defn- table-query [card-name]
+  (-> common.pulses/table-query
+      (update :select (partial into
+                               [[:card.id :card_id]
+                                [:card.name :card_name]]))
+      (update :left-join into [:pulse_card          [:= :pulse.id :pulse_card.pulse_id]
+                               [:report_card :card] [:= :pulse_card.card_id :card.id]])
+      (update :where (fn [where]
+                       (into
+                        where
+                        (filter some?)
+                        ;; make sure the pulse_card actually exists.
+                        [[:not= :pulse_card.card_id nil]
+                         [:= :pulse.dashboard_id nil]
+                         ;; if `pulse.alert_condition` is non-NULL then the Pulse is an Alert
+                         [:not= :pulse.alert_condition nil]
+                         (when-not (str/blank? card-name)
+                           [:like :%lower.card.name (str \% (str/lower-case card-name) \%)])])))
+      (assoc :order-by [[:%lower.card.name :asc]
+                        ;; Newest first. ID instead of `created_at` because the column is currently only
+                        ;; second-resolution for MySQL which busts our tests
+                        [:channel.id :desc]])))
+
+(def ^:private ^{:arglists '([row-map])} row-map->vec
+  (apply juxt (map first table-metadata)))
+
+(defn- post-process-row [row]
+  (-> (zipmap table-query-columns row)
+      common.pulses/post-process-row-map
+      row-map->vec))
+
+;; with optional param `card-name`, only show subscriptions matching card name.
+(defmethod audit.i/internal-query ::table
+  ([query-type]
+   (audit.i/internal-query query-type nil))
+
+  ([_ card-name]
+   {:metadata table-metadata
+    :results  (common/reducible-query (table-query card-name))
+    :xform    (map post-process-row)}))

enterprise/backend/src/metabase_enterprise/audit_app/pages/common.clj

@@ -19,6 +19,7 @@
             [metabase.query-processor.timezone :as qp.tz]
             [metabase.util :as u]
             [metabase.util.honeysql-extensions :as hx]
+            [metabase.util.i18n :refer [tru]]
             [metabase.util.urls :as urls]
             [schema.core :as s]
             [toucan.db :as db]))
@@ -91,12 +92,20 @@
     (fn []
       (timezone (mdb/db-type) (db/connection)))))
 
+(defn- compile-honeysql [driver honeysql-query]
+  (try
+    (let [honeysql-query (cond-> honeysql-query
+                           ;; MySQL 5.x does not support CTEs, so convert them to subselects instead
+                           (= driver :mysql) CTEs->subselects)]
+      (db/honeysql->sql (add-default-params honeysql-query)))
+    (catch Throwable e
+      (throw (ex-info (tru "Error compiling audit query: {0}" (ex-message e))
+                      {:driver driver, :honeysql-query honeysql-query}
+                      e)))))
+
 (defn- reduce-results* [honeysql-query context rff init]
   (let [driver         (mdb/db-type)
-        honeysql-query (cond-> honeysql-query
-                         ;; MySQL 5.x does not support CTEs, so convert them to subselects instead
-                         (= driver :mysql) CTEs->subselects)
-        [sql & params] (db/honeysql->sql (add-default-params honeysql-query))
+        [sql & params] (compile-honeysql driver honeysql-query)
         canceled-chan  (context/canceled-chan context)]
     ;; MySQL driver normalizies timestamps. Setting `*results-timezone-id-override*` is a shortcut
     ;; instead of mocking up a chunk of regular QP pipeline.
@@ -112,7 +121,14 @@
             (reduce rf init (sql-jdbc.execute/reducible-rows driver rs rsmeta canceled-chan))))
         (catch InterruptedException e
           (a/>!! canceled-chan :cancel)
-          (throw e))))))
+          (throw e))
+        (catch Throwable e
+          (throw (ex-info (tru "Error running audit query: {0}" (ex-message e))
+                          {:driver         driver
+                           :honeysql-query honeysql-query
+                           :sql            sql
+                           :params         params}
+                          e)))))))
 
 (defn reducible-query
   "Return a function with the signature

enterprise/backend/src/metabase_enterprise/audit_app/pages/common/pulses.clj

+(ns metabase-enterprise.audit-app.pages.common.pulses
+  "Shared code for [[metabase-enterprise.audit-app.pages.dashboard-subscriptions]]
+  and [[metabase-enterprise.audit-app.pages.alerts]]."
+  (:require [cheshire.core :as json]
+            [clojure.tools.logging :as log]
+            [metabase.models.collection :as collection]
+            [metabase.util.cron :as u.cron]
+            [metabase.util.honeysql-extensions :as hx]
+            [metabase.util.i18n :refer [trs tru]]))
+
+(def table-metadata
+  "Common Metadata for the columns returned by both the [[metabase-enterprise.audit-app.pages.dashboard-subscriptions]]
+  and [[metabase-enterprise.audit-app.pages.alerts]] audit queries. (These respective queries also return their own
+  additional columns.)"
+  [[:pulse_id          {:display_name "Pulse ID",      :base_type :type/Integer}]
+   [:recipients        {:display_name "Recipients",    :base_type :type/Integer}]
+   [:subscription_type {:display_name "Type",          :base_type :type/Text}]
+   [:collection_id     {:display_name "Collection ID", :base_type :type/Integer,        :remapped_to :collection_name}]
+   [:collection_name   {:display_name "Collection",    :base_type :type/Text,           :remapped_from :collection_id}]
+   [:frequency         {:display_name "Frequency",     :base_type :type/Text}]
+   [:creator_id        {:display_name "Created By ID", :base_type :type/Integer,        :remapped_to :creator_name}]
+   [:creator_name      {:display_name "Created By",    :base_type :type/Text,           :remapped_from :creator_id}]
+   [:created_at        {:display_name "Created At",    :base_type :type/DateTimeWithTZ}]
+   [:num_filters       {:display_name "Filters",       :base_type :type/Integer}]])
+
+(def table-query-columns
+  "Keyword names of columns returned by the queries by both
+  the [[metabase-enterprise.audit-app.pages.dashboard-subscriptions]] and [[metabase-enterprise.audit-app.pages.alerts]] audit
+  queries."
+  [:pulse_id
+   :num_user_recipients
+   :channel_id
+   :channel_details
+   :subscription_type
+   :collection_id
+   :collection_name
+   :schedule_type
+   :schedule_hour
+   :schedule_day
+   :schedule_frame
+   :creator_id
+   :creator_name
+   :created_at
+   :pulse_parameters])
+
+(def table-query
+  "Common HoneySQL base query for both the [[metabase-enterprise.audit-app.pages.dashboard-subscriptions]]
+  and [[metabase-enterprise.audit-app.pages.alerts]] audit queries. (The respective implementations tweak this query and
+  add additional columns, filters, and order-by clauses.)"
+  {:with      [[:user_recipients {:select   [[:recipient.pulse_channel_id :channel_id]
+                                             [:%count.* :count]]
+                                  :from     [[:pulse_channel_recipient :recipient]]
+                                  :group-by [:channel_id]}]]
+   :select    [[:pulse.id :pulse_id]
+               [:user_recipients.count :num_user_recipients]
+               [:channel.id :channel_id]
+               [:channel.details :channel_details]
+               [:channel.channel_type :subscription_type]
+               [:collection.id :collection_id]
+               [:collection.name :collection_name]
+               :channel.schedule_type
+               :channel.schedule_hour
+               :channel.schedule_day
+               :channel.schedule_frame
+               [:creator.id :creator_id]
+               [(hx/concat :creator.first_name (hx/literal " ") :creator.last_name) :creator_name]
+               [:channel.created_at :created_at]
+               [:pulse.parameters :pulse_parameters]]
+   :from      [[:pulse_channel :channel]]
+   :left-join [:pulse                         [:= :channel.pulse_id :pulse.id]
+               :collection                    [:= :pulse.collection_id :collection.id]
+               [:core_user :creator]          [:= :pulse.creator_id :creator.id]
+               :user_recipients               [:= :channel.id :user_recipients.channel_id]]
+   :where     [:and
+               [:not= :pulse.archived true]
+               [:= :channel.enabled true]]})
+
+(defn- describe-frequency [row]
+  (-> (select-keys row [:schedule_type :schedule_hour :schedule_day :schedule_frame])
+      u.cron/schedule-map->cron-string
+      u.cron/describe-cron-string))
+
+(defn- describe-recipients
+  "Return the number of recipients for email `PulseChannel`s. Includes both User recipients (represented by
+  `PulseChannelRecipient` rows) and plain email recipients (stored directly in the `PulseChannel` `:details`). Returns
+  `nil` for Slack channels."
+  [{channel-id        :channel_id
+    subscription-type :subscription_type
+    channel-details   :channel_details
+    num-recipients    :num_user_recipients}]
+  (let [details (json/parse-string channel-details true)]
+    (when (= (keyword subscription-type) :email)
+      ((fnil + 0 0) num-recipients (count (:emails details))))))
+
+(defn- pulse-parameter-count [{pulse-parameters :pulse_parameters}]
+  (if-let [params (try
+                    (some-> pulse-parameters (json/parse-string true))
+                    (catch Throwable e
+                      (log/error e (trs "Error parsing Pulse parameters: {0}" (ex-message e)))
+                      nil))]
+    (count params)
+    0))
+
+(defn- root-collection-name []
+  (:name (collection/root-collection-with-ui-details nil)))
+
+(defn post-process-row-map
+  "Post-process a `row` **map** for the subscription and alert audit page tables. Get this map by doing something like
+  this:
+
+    (zipmap table-query-columns row-vector)
+
+  This map should contain at least the keys in [[table-query-columns]] (provided by the common [[table-query]]). After
+  calling this function, you'll need to convert the row map back to a vector; something like
+
+    (apply juxt (map first table-metadata))
+
+  should do the trick."
+  [row]
+  {:pre [(map? row)]}
+  (-> row
+      (assoc :frequency  (describe-frequency row)
+             :recipients (describe-recipients row)
+             :num_filters (pulse-parameter-count row))
+      (update :subscription_type (fn [subscription-type]
+                                   (case (keyword subscription-type)
+                                     :email (tru "Email")
+                                     :slack (tru "Slack")
+                                     subscription-type)))
+      (update :collection_name #(or % (root-collection-name)))))

enterprise/backend/src/metabase_enterprise/audit_app/pages/dashboard_subscriptions.clj

+(ns metabase-enterprise.audit-app.pages.dashboard-subscriptions
+  (:require [clojure.string :as str]
+            [metabase-enterprise.audit-app.interface :as audit.i]
+            [metabase-enterprise.audit-app.pages.common :as common]
+            [metabase-enterprise.audit-app.pages.common.pulses :as common.pulses]))
+
+(def ^:private table-metadata
+  (into
+   [[:dashboard_id   {:display_name "Dashboard ID",  :base_type :type/Integer, :remapped_to :dashboard_name}]
+    [:dashboard_name {:display_name "Dashboard Name" :base_type :type/Text,    :remapped_from :dashboard_id}]]
+   common.pulses/table-metadata))
+
+(def ^:private table-query-columns
+  (into
+   [:dashboard_id
+    :dashboard_name]
+   common.pulses/table-query-columns))
+
+(defn- table-query [dashboard-name]
+  (-> common.pulses/table-query
+      (update :select (partial into
+                               [[:dashboard.id :dashboard_id]
+                                [:dashboard.name :dashboard_name]]))
+      (update :left-join into [[:report_dashboard :dashboard] [:= :pulse.dashboard_id :dashboard.id]])
+      (update :where (fn [where]
+                       (into
+                        where
+                        (filter some?)
+                        [[:not= :pulse.dashboard_id nil]
+                         (when-not (str/blank? dashboard-name)
+                           [:like :%lower.dashboard.name (str \% (str/lower-case dashboard-name) \%)])])))
+      (assoc :order-by [[:%lower.dashboard.name :asc]
+                        ;; Newest first. ID instead of `created_at` because the column is currently only
+                        ;; second-resolution for MySQL which busts our tests
+                        [:channel.id :desc]])))
+
+(def ^:private ^{:arglists '([row-map])} row-map->vec
+  (apply juxt (map first table-metadata)))
+
+(defn- post-process-row [row]
+  (-> (zipmap table-query-columns row)
+      common.pulses/post-process-row-map
+      row-map->vec))
+
+;; with optional param `dashboard-name`, only show subscriptions matching dashboard name.
+(defmethod audit.i/internal-query ::table
+  ([query-type]
+   (audit.i/internal-query query-type nil))
+
+  ([_ dashboard-name]
+   {:metadata table-metadata
+    :results  (common/reducible-query (table-query dashboard-name))
+    :xform    (map post-process-row)}))

enterprise/backend/test/metabase_enterprise/audit_app/pages/alerts_test.clj

+(ns metabase-enterprise.audit-app.pages.alerts-test
+  (:require [clojure.string :as str]
+            [clojure.test :refer :all]
+            [metabase-enterprise.audit-app.pages.alerts :as audit.alerts]
+            [metabase.models :refer [Card Collection Pulse PulseCard PulseChannel PulseChannelRecipient]]
+            [metabase.public-settings.premium-features-test :as premium-features-test]
+            [metabase.query-processor :as qp]
+            [metabase.test :as mt]
+            [metabase.util :as u]
+            [toucan.db :as db]))
+
+(defn- alerts [card-name]
+  (mt/with-test-user :crowberto
+    (premium-features-test/with-premium-features #{:audit-app}
+      (qp/process-query
+       {:type :internal
+        :fn   (u/qualified-name ::audit.alerts/table)
+        :args [card-name]}))))
+
+(deftest table-test
+  (is (= []
+         (mt/rows (alerts (mt/random-name)))))
+  (let [card-name (mt/random-name)]
+    (mt/with-temp Collection [{collection-id :id, collection-name :name}]
+      ;; test with both the Root Collection and a non-Root Collection
+      (doseq [{:keys [collection-id collection-name]} [{:collection-id   collection-id
+                                                        :collection-name collection-name}
+                                                       {:collection-id   nil
+                                                        :collection-name "Our analytics"}]]
+        (testing (format "Collection = %d %s" collection-id collection-name)
+          (mt/with-temp* [Card                  [{card-id :id}      {:name          card-name
+                                                                     :collection_id collection-id}]
+                          Pulse                 [{pulse-id :id}     {:collection_id   collection-id
+                                                                     :alert_condition "rows"}]
+                          PulseCard             [_                  {:card_id  card-id
+                                                                     :pulse_id pulse-id}]
+                          PulseChannel          [{channel-id :id}   {:pulse_id       pulse-id
+                                                                     :channel_type   "email"
+                                                                     :details        {:emails ["amazing@fake.com"]}
+                                                                     :schedule_type  "monthly"
+                                                                     :schedule_frame "first"
+                                                                     :schedule_day   "mon"
+                                                                     :schedule_hour  8}]
+                          PulseChannelRecipient [_                  {:pulse_channel_id channel-id
+                                                                     :user_id          (mt/user->id :rasta)}]
+                          PulseChannel          [{channel-2-id :id} {:pulse_id      pulse-id
+                                                                     :channel_type  "slack"
+                                                                     :details       {:channel "#wow"}
+                                                                     :schedule_type "hourly"}]]
+            (is (= {:columns ["card_id"
+                              "card_name"
+                              "pulse_id"
+                              "recipients"
+                              "subscription_type"
+                              "collection_id"
+                              "collection_name"
+                              "frequency"
+                              "creator_id"
+                              "creator_name"
+                              "created_at"
+                              "num_filters"]
+                    ;; sort by newest first.
+                    :rows    [[card-id
+                               card-name
+                               pulse-id
+                               nil
+                               "Slack"
+                               collection-id
+                               collection-name
+                               "Every hour"
+                               (mt/user->id :rasta)
+                               "Rasta Toucan"
+                               (db/select-one-field :created_at PulseChannel :id channel-2-id)
+                               0]
+                              [card-id
+                               card-name
+                               pulse-id
+                               2
+                               "Email"
+                               collection-id
+                               collection-name
+                               "At 8:00 AM, on the first Tuesday of the month"
+                               (mt/user->id :rasta)
+                               "Rasta Toucan"
+                               (db/select-one-field :created_at PulseChannel :id channel-id)
+                               0]]}
+                   (mt/rows+column-names
+                    (alerts (str/join (rest (butlast card-name)))))))))))))

enterprise/backend/test/metabase_enterprise/audit_app/pages/dashboard_subscriptions_test.clj

+(ns metabase-enterprise.audit-app.pages.dashboard-subscriptions-test
+  (:require [clojure.string :as str]
+            [clojure.test :refer :all]
+            [metabase-enterprise.audit-app.pages.dashboard-subscriptions :as audit.dashboard-subscriptions]
+            [metabase.models :refer [Collection Dashboard Pulse PulseChannel PulseChannelRecipient]]
+            [metabase.public-settings.premium-features-test :as premium-features-test]
+            [metabase.query-processor :as qp]
+            [metabase.test :as mt]
+            [metabase.util :as u]
+            [toucan.db :as db]))
+
+(defn- dashboard-subscriptions [dashboard-name]
+  (mt/with-test-user :crowberto
+    (premium-features-test/with-premium-features #{:audit-app}
+      (qp/process-query
+       {:type :internal
+        :fn   (u/qualified-name ::audit.dashboard-subscriptions/table)
+        :args [dashboard-name]}))))
+
+(deftest table-test
+  (is (= []
+         (mt/rows (dashboard-subscriptions (mt/random-name)))))
+  (let [dashboard-name (mt/random-name)]
+    (mt/with-temp Collection [{collection-id :id, collection-name :name}]
+      ;; test with both the Root Collection and a non-Root Collection
+      (doseq [{:keys [collection-id collection-name]} [{:collection-id   collection-id
+                                                        :collection-name collection-name}
+                                                       {:collection-id   nil
+                                                        :collection-name "Our analytics"}]]
+        (testing (format "Collection = %d %s" collection-id collection-name)
+          (mt/with-temp* [Dashboard             [{dashboard-id :id} {:name          dashboard-name
+                                                                     :collection_id collection-id}]
+                          Pulse                 [{pulse-id :id}     {:dashboard_id  dashboard-id
+                                                                     :collection_id collection-id}]
+                          PulseChannel          [{channel-id :id}   {:pulse_id       pulse-id
+                                                                     :channel_type   "email"
+                                                                     :details        {:emails ["amazing@fake.com"]}
+                                                                     :schedule_type  "monthly"
+                                                                     :schedule_frame "first"
+                                                                     :schedule_day   "mon"
+                                                                     :schedule_hour  8}]
+                          PulseChannelRecipient [_                  {:pulse_channel_id channel-id
+                                                                     :user_id          (mt/user->id :rasta)}]
+                          PulseChannel          [{channel-2-id :id} {:pulse_id      pulse-id
+                                                                     :channel_type  "slack"
+                                                                     :details       {:channel "#wow"}
+                                                                     :schedule_type "hourly"}]]
+            (is (= {:columns ["dashboard_id"
+                              "dashboard_name"
+                              "pulse_id"
+                              "recipients"
+                              "subscription_type"
+                              "collection_id"
+                              "collection_name"
+                              "frequency"
+                              "creator_id"
+                              "creator_name"
+                              "created_at"
+                              "num_filters"]
+                    ;; sort by newest first.
+                    :rows    [[dashboard-id
+                               dashboard-name
+                               pulse-id
+                               nil
+                               "Slack"
+                               collection-id
+                               collection-name
+                               "Every hour"
+                               (mt/user->id :rasta)
+                               "Rasta Toucan"
+                               (db/select-one-field :created_at PulseChannel :id channel-2-id)
+                               0]
+                              [dashboard-id
+                               dashboard-name
+                               pulse-id
+                               2
+                               "Email"
+                               collection-id
+                               collection-name
+                               "At 8:00 AM, on the first Tuesday of the month"
+                               (mt/user->id :rasta)
+                               "Rasta Toucan"
+                               (db/select-one-field :created_at PulseChannel :id channel-id)
+                               0]]}
+                   (mt/rows+column-names
+                    (dashboard-subscriptions (str/join (rest (butlast dashboard-name)))))))))))))

enterprise/backend/test/metabase_enterprise/audit_app/pages_test.clj

@@ -135,7 +135,9 @@
                :error-filter      "a"
                :db-filter         "PU"
                :sort-column       "card.id"
-               :sort-direction    "desc"))}))
+               :sort-direction    "desc"
+               :dashboard-name    "wow"
+               :card-name         "Credit Card"))}))
 
 (defn- do-tests-for-query-type
   "Run test(s) for the internal query named by `query-type`. Runs one test for each map returned