From 82742ccc1f14c546958fca2d8b3a99a78b9c72ab Mon Sep 17 00:00:00 2001
From: Howon Lee <hlee.howon@gmail.com>
Date: Thu, 23 Sep 2021 12:33:24 -0700
Subject: [PATCH] Tools to fix broken questions timeout fix (#17985)
Previous implementation was dog slow. This one is faster
---
.../audit_app/pages/common/cards.clj | 22 ++++++++++
.../audit_app/pages/queries.clj | 40 +++++++------------
2 files changed, 36 insertions(+), 26 deletions(-)
diff --git a/enterprise/backend/src/metabase_enterprise/audit_app/pages/common/cards.clj b/enterprise/backend/src/metabase_enterprise/audit_app/pages/common/cards.clj
index 247c268a0be..3bc09218760 100644
--- a/enterprise/backend/src/metabase_enterprise/audit_app/pages/common/cards.clj
+++ b/enterprise/backend/src/metabase_enterprise/audit_app/pages/common/cards.clj
@@ -25,6 +25,22 @@
:group-by [:card_id]}
(common/add-45-days-clause :started_at))])
+(def latest-qe
+ "HoneySQL for a CTE to get latest QueryExecution for a Card."
+ [:latest_qe {:select [:query_execution.card_id :error :query_execution.started_at]
+ :from [:query_execution]
+ :join [[{:select [:card_id [:%max.started_at :started_at]]
+ :from [:query_execution]
+ :group-by [:card_id]} :inner_qe]
+ [:= :query_execution.started_at :inner_qe.started_at]]}])
+
+(def query-runs
+ "HoneySQL for a CTE to include the total number of queries for each Card forever."
+ [:query_runs {:select [:card_id
+ [:%count.* :count]]
+ :from [:query_execution]
+ :group-by [:card_id]}])
+
(def query-runs-45
"HoneySQL for a CTE to include the total number of queries for each Card for 45 days."
[:query_runs (-> {:select [:card_id
@@ -33,6 +49,12 @@
:group-by [:card_id]}
(common/add-45-days-clause :started_at))])
+(def dashboards-count
+ "HoneySQL for a CTE to enumerate the dashboards for a Card."
+ [:dash_card {:select [:card_id [:%count.* :count]]
+ :from [:report_dashboardcard]
+ :group-by [:card_id]}])
+
(def views
"HoneySQL for a CTE to include the total view count for each Card."
[:card_views {:select [[:model_id :card_id]
diff --git a/enterprise/backend/src/metabase_enterprise/audit_app/pages/queries.clj b/enterprise/backend/src/metabase_enterprise/audit_app/pages/queries.clj
index 6712084d0df..4bbe29faec4 100644
--- a/enterprise/backend/src/metabase_enterprise/audit_app/pages/queries.clj
+++ b/enterprise/backend/src/metabase_enterprise/audit_app/pages/queries.clj
@@ -53,14 +53,6 @@
:order-by [[:avg_running_time :desc]]
:limit 10})})
-(def latest-qe-subq
- "QE subquery for only getting the latest QE. Needed to make the QE table blank out properly after running."
- [:not [:exists {:select [1]
- :from [[:query_execution :qe1]]
- :where [:and
- [:= :card.id :qe1.card_id]
- [:> :qe1.started_at :qe.started_at]]}]])
-
;; List of all failing questions
(defmethod audit.i/internal-query ::bad-table
([_]
@@ -89,40 +81,36 @@
[:updated_at {:display_name "Updated At", :base_type :type/DateTime}]]
:results (common/reducible-query
(->
- {:select [[:card.id :card_id]
+ {:with [cards/query-runs
+ cards/latest-qe
+ cards/dashboards-count]
+ :select [[:card.id :card_id]
[:card.name :card_name]
- [(hsql/call :concat (hsql/call :substring :qe.error 0 60) "...") :error_substr]
+ [(hsql/call :concat (hsql/call :substring :latest_qe.error 0 60) "...") :error_substr]
:collection_id
[:coll.name :collection_name]
:card.database_id
[:db.name :database_name]
:card.table_id
[:t.name :table_name]
- [(hsql/call :max :qe.started_at) :last_run_at]
- [:%distinct-count.qe.id :total_runs]
- [:%distinct-count.dash_card.card_id :num_dashboards]
+ [:latest_qe.started_at :last_run_at]
+ [:query_runs.count :total_runs]
+ [:dash_card.count :num_dashboards]
[:card.creator_id :user_id]
[(common/user-full-name :u) :user_name]
- [(hsql/call :max :card.updated_at) :updated_at]]
+ [:card.updated_at :updated_at]]
:from [[:report_card :card]]
:left-join [[:collection :coll] [:= :card.collection_id :coll.id]
[:metabase_database :db] [:= :card.database_id :db.id]
[:metabase_table :t] [:= :card.table_id :t.id]
[:core_user :u] [:= :card.creator_id :u.id]
- [:report_dashboardcard :dash_card] [:= :card.id :dash_card.card_id]
- [:query_execution :qe] [:and [:= :card.id :qe.card_id]
- latest-qe-subq]]
- :group-by [(common/user-full-name :u)
- :card.id
- :card.creator_id
- :coll.name
- :db.name
- :t.name
- :qe.error]
+ :latest_qe [:= :card.id :latest_qe.card_id]
+ :query_runs [:= :card.id :query_runs.card_id]
+ :dash_card [:= :card.id :dash_card.card_id]]
:where [:and
[:= :card.archived false]
- [:<> :qe.error nil]]}
- (common/add-search-clause error-filter :qe.error)
+ [:<> :latest_qe.error nil]]}
+ (common/add-search-clause error-filter :latest_qe.error)
(common/add-search-clause db-filter :db.name)
(common/add-search-clause collection-filter :coll.name)
(common/add-sort-clause
--
GitLab