diff --git a/enterprise/backend/src/metabase_enterprise/audit_db.clj b/enterprise/backend/src/metabase_enterprise/audit_db.clj
new file mode 100644
index 0000000000000000000000000000000000000000..a195c996eae8048a72b491da47c250fbe9a5d747
--- /dev/null
+++ b/enterprise/backend/src/metabase_enterprise/audit_db.clj
@@ -0,0 +1,58 @@
+(ns metabase-enterprise.audit-db
+ (:require [metabase.db.env :as mdb.env]
+ [metabase.models.database :refer [Database]]
+ [metabase.public-settings.premium-features :refer [defenterprise]]
+ [metabase.util :as u]
+ [metabase.util.log :as log]
+ [toucan2.core :as t2]))
+
+(def ^:private default-admin-db-id 13371337)
+
+(defn- install-database!
+ "Creates the audit db, a clone of the app db used for auditing purposes.
+
+ - This uses a weird ID because some tests are hardcoded to look for database with ID = 2, and inserting an extra db
+ throws that off since the IDs are sequential...
+
+ - In the unlikely case that a user has many many databases in Metabase, and ensure there can Never be a collision, we
+ do a quick check here and pick a new ID if it would have collided. Similar to finding an open port number."
+ ([engine] (install-database! engine default-admin-db-id))
+ ([engine id]
+ (if (t2/select-one Database :id id)
+ (install-database! engine (inc id))
+ (t2/insert! Database {:is_audit true
+ :id default-admin-db-id
+ :name "Audit Database"
+ :description "Internal Audit DB used to power metabase analytics."
+ :engine engine
+ :is_full_sync true
+ :is_on_demand false
+ :creator_id nil
+ :auto_run_queries true}))))
+
+(defn ensure-db-installed!
+ "Called on app startup to ensure the existance of the audit db in enterprise apps.
+
+ The return values indicate what action was taken."
+ []
+ (let [audit-db (t2/select-one Database :is_audit true)]
+ (cond
+ (nil? audit-db)
+ (u/prog1 ::installed
+ (log/info "Audit DB does not exist, Installing...")
+ (install-database! mdb.env/db-type))
+
+ (not= mdb.env/db-type (:engine audit-db))
+ (u/prog1 ::updated
+ (log/infof "Updating the Audit DB engine to %s." (name mdb.env/db-type))
+ (t2/update! Database :is_audit true {:engine mdb.env/db-type})
+ (ensure-db-installed!))
+
+ :else
+ ::no-op)))
+
+(defenterprise ensure-audit-db-installed!
+ "EE implementation of `ensure-db-installed!`."
+ :feature :any
+ []
+ (ensure-db-installed!))
diff --git a/enterprise/backend/test/metabase_enterprise/audit_db_test.clj b/enterprise/backend/test/metabase_enterprise/audit_db_test.clj
new file mode 100644
index 0000000000000000000000000000000000000000..86c3103e7817bd40ccf3d18c1d4fcf43d860e319
--- /dev/null
+++ b/enterprise/backend/test/metabase_enterprise/audit_db_test.clj
@@ -0,0 +1,23 @@
+(ns metabase-enterprise.audit-db-test
+ (:require [clojure.test :refer [deftest is]]
+ [metabase-enterprise.audit-db :as audit-db]
+ [metabase.models.database :refer [Database]]
+ [metabase.util.log :as log]
+ [toucan2.core :as t2]))
+
+(deftest audit-db-is-installed-then-left-alone
+ (let [original-audit-db (t2/select-one Database :is_audit true)]
+ (try
+ (t2/delete! Database :is_audit true)
+ (is (= :metabase-enterprise.audit-db/installed (audit-db/ensure-db-installed!)))
+ (is (= :metabase-enterprise.audit-db/no-op (audit-db/ensure-db-installed!)))
+
+ (t2/update! Database :is_audit true {:engine "datomic"})
+ (is (= :metabase-enterprise.audit-db/updated (audit-db/ensure-db-installed!)))
+ (is (= :metabase-enterprise.audit-db/no-op (audit-db/ensure-db-installed!)))
+
+ (finally
+ (t2/delete! Database :is_audit true)
+ (when original-audit-db
+ (log/fatal (str "Original Audit DB: " (pr-str original-audit-db)))
+ (#'t2/insert! Database original-audit-db))))))
diff --git a/resources/migrations/000_migrations.yaml b/resources/migrations/000_migrations.yaml
index c43b56e9f56c276d8117f2f4a0090ae7cfe8679b..0c3ead6e63c31f8fb3006b50468438d0dd8f67df 100644
--- a/resources/migrations/000_migrations.yaml
+++ b/resources/migrations/000_migrations.yaml
@@ -14591,6 +14591,22 @@ databaseChangeLog:
constraints:
nullable: false
+ - changeSet:
+ id: v47.00-015
+ author: escherize
+ comment: added 0.47.0 - Add is_audit to metabase_database
+ changes:
+ - addColumn:
+ tableName: metabase_database
+ columns:
+ - column:
+ name: is_audit
+ type: boolean
+ defaultValueBoolean: false
+ remarks: 'Only the app db, visible to admins via auditing should have this set true.'
+ constraints:
+ nullable: false
+
- changeSet:
id: v47.00-016
author: calherres
diff --git a/src/metabase/api/database.clj b/src/metabase/api/database.clj
index c75119cdfab86d837a6e3db87d136ae789445265..e079fca4631f01715b88868bef2e2ea1bb84f67e 100644
--- a/src/metabase/api/database.clj
+++ b/src/metabase/api/database.clj
@@ -43,6 +43,7 @@
[metabase.util.honey-sql-2 :as h2x]
[metabase.util.i18n :refer [deferred-tru trs tru]]
[metabase.util.log :as log]
+ [metabase.util.malli :as mu]
[metabase.util.malli.schema :as ms]
[metabase.util.schema :as su]
[schema.core :as s]
@@ -231,7 +232,8 @@
include-saved-questions-tables?
include-editable-data-model?
exclude-uneditable-details?]}]
- (let [dbs (t2/select Database {:order-by [:%lower.name :%lower.engine]})
+ (let [dbs (t2/select Database {:where [:= :is_audit false]
+ :order-by [:%lower.name :%lower.engine]})
filter-by-data-access? (not (or include-editable-data-model? exclude-uneditable-details?))]
(cond-> (add-native-perms-info dbs)
include-tables? add-tables
@@ -240,14 +242,7 @@
filter-by-data-access? (#(filter mi/can-read? %))
include-saved-questions-db? (add-saved-questions-virtual-database :include-tables? include-saved-questions-tables?))))
-(def FetchAllIncludeValues
- "Schema for matching the include parameter of the GET / endpoint"
- (su/with-api-error-message
- (s/maybe (s/eq "tables"))
- (deferred-tru "include must be either empty or the value 'tables'")))
-
-#_{:clj-kondo/ignore [:deprecated-var]}
-(api/defendpoint-schema GET "/"
+(api/defendpoint GET "/"
"Fetch all `Databases`.
* `include=tables` means we should hydrate the Tables belonging to each DB. Default: `false`.
@@ -268,32 +263,28 @@
* `exclude_uneditable_details` will only include DBs for which the current user can edit the DB details. Has no
effect unless Enterprise Edition code is available and the advanced-permissions feature is enabled."
[include_tables include_cards include saved include_editable_data_model exclude_uneditable_details]
- {include_tables (s/maybe su/BooleanString)
- include_cards (s/maybe su/BooleanString)
- include FetchAllIncludeValues
- saved (s/maybe su/BooleanString)
- include_editable_data_model (s/maybe su/BooleanString)
- exclude_uneditable_details (s/maybe su/BooleanString)}
+ {include_tables [:maybe :boolean]
+ include_cards [:maybe :boolean]
+ include (mu/with-api-error-message
+ [:maybe [:= "tables"]]
+ (deferred-tru "include must be either empty or the value 'tables'"))
+ saved [:maybe :boolean]
+ include_editable_data_model [:maybe :boolean]
+ exclude_uneditable_details [:maybe :boolean]}
(when (and config/is-dev?
(or include_tables include_cards))
;; don't need to i18n since this is dev-facing only
(log/warn "GET /api/database?include_tables and ?include_cards are deprecated."
"Prefer using ?include=tables and ?saved=true instead."))
- (let [include-tables? (cond
- (seq include) (= include "tables")
- (seq include_tables) (Boolean/parseBoolean include_tables))
- include-saved-questions-db? (cond
- (seq saved) (Boolean/parseBoolean saved)
- (seq include_cards) (Boolean/parseBoolean include_cards))
+ (let [include-tables? (or (= include "tables") include_tables)
+ include-saved-questions-db? (or saved include_cards)
include-saved-questions-tables? (when include-saved-questions-db?
- (if (seq include_cards)
- true
- include-tables?))
+ (if include_cards true include-tables?))
db-list-res (or (dbs-list :include-tables? include-tables?
:include-saved-questions-db? include-saved-questions-db?
:include-saved-questions-tables? include-saved-questions-tables?
- :include-editable-data-model? (Boolean/parseBoolean include_editable_data_model)
- :exclude-uneditable-details? (Boolean/parseBoolean exclude_uneditable_details))
+ :include-editable-data-model? include_editable_data_model
+ :exclude-uneditable-details? exclude_uneditable_details)
[])]
{:data db-list-res
:total (count db-list-res)}))
@@ -759,21 +750,21 @@
;; no error, proceed with creation. If record is inserted successfuly, publish a `:database-create` event.
;; Throw a 500 if nothing is inserted
(u/prog1 (api/check-500 (first (t2/insert-returning-instances!
- Database
- (merge
- {:name name
- :engine engine
- :details details-or-error
- :is_full_sync is-full-sync?
- :is_on_demand (boolean is_on_demand)
- :cache_ttl cache_ttl
- :creator_id api/*current-user-id*}
- (sync.schedules/schedule-map->cron-strings
- (if (:let-user-control-scheduling details)
- (sync.schedules/scheduling schedules)
- (sync.schedules/default-randomized-schedule)))
- (when (some? auto_run_queries)
- {:auto_run_queries auto_run_queries})))))
+ Database
+ (merge
+ {:name name
+ :engine engine
+ :details details-or-error
+ :is_full_sync is-full-sync?
+ :is_on_demand (boolean is_on_demand)
+ :cache_ttl cache_ttl
+ :creator_id api/*current-user-id*}
+ (sync.schedules/schedule-map->cron-strings
+ (if (:let-user-control-scheduling details)
+ (sync.schedules/scheduling schedules)
+ (sync.schedules/default-randomized-schedule)))
+ (when (some? auto_run_queries)
+ {:auto_run_queries auto_run_queries})))))
(events/publish-event! :database-create <>)
(snowplow/track-event! ::snowplow/database-connection-successful
api/*current-user-id*
diff --git a/src/metabase/core.clj b/src/metabase/core.clj
index 441e2cf6cf07ed1f8ffda3a84a53803f6c854cc8..c07cee10f244bdc657280470f16de0442837feca 100644
--- a/src/metabase/core.clj
+++ b/src/metabase/core.clj
@@ -17,6 +17,7 @@
[metabase.plugins :as plugins]
[metabase.plugins.classloader :as classloader]
[metabase.public-settings :as public-settings]
+ [metabase.public-settings.premium-features :refer [defenterprise]]
[metabase.sample-data :as sample-data]
[metabase.server :as server]
[metabase.server.handler :as handler]
@@ -89,6 +90,12 @@
(prometheus/shutdown!)
(log/info (trs "Metabase Shutdown COMPLETE")))
+(defenterprise ensure-audit-db-installed!
+ "OSS implementation of `audit-db/ensure-db-installed!`, which is an enterprise feature, so does nothing in the OSS
+ version."
+ metabase-enterprise.audit-db
+ [])
+
(defn- init!*
"General application initialization function which should be run once at application startup."
[]
@@ -133,6 +140,7 @@
;; otherwise update if appropriate
(sample-data/update-sample-database-if-needed!))
(init-status/set-progress! 0.9))
+ (ensure-audit-db-installed!)
;; start scheduler at end of init!
(task/start-scheduler!)
(init-status/set-complete!)
diff --git a/src/metabase/db/env.clj b/src/metabase/db/env.clj
index f08d521d23914881b152de8bcc8b588793a81c51..c328884f3b5e002efdbedf9cfd25ddb245f8c191 100644
--- a/src/metabase/db/env.clj
+++ b/src/metabase/db/env.clj
@@ -26,7 +26,8 @@
[metabase.config :as config]
[metabase.db.data-source :as mdb.data-source]
[metabase.util :as u]
- [metabase.util.log :as log]))
+ [metabase.util.log :as log]
+ [metabase.util.malli :as mu]))
(set! *warn-on-reflection* true)
@@ -39,13 +40,11 @@
"postgresql" :postgres
(keyword subprotocol)))))
-(defn- env->db-type
+(mu/defn ^:private env->db-type :- [:enum :postgres :mysql :h2]
[{:keys [mb-db-connection-uri mb-db-type]}]
- {:post [(#{:postgres :mysql :h2} %)]}
(or (some-> mb-db-connection-uri raw-connection-string->type)
mb-db-type))
-
;;;; [[env->DataSource]]
(defn- get-db-file
@@ -132,7 +131,8 @@
:mb-db-pass (config/config-str :mb-db-pass)}
(env-defaults db-type)))
-(def ^:private env
+(def env
+ "Metabase Datatbase environment. Used to setup *application-db* and audit-db for enterprise users."
(env* (config/config-kw :mb-db-type)))
(def db-type
diff --git a/src/metabase/driver/sql_jdbc/connection.clj b/src/metabase/driver/sql_jdbc/connection.clj
index 6598ebf8d7b373074c17f42ad0a184035a26be66..082600692401139e53aaebb68771389e1aa6b0d7 100644
--- a/src/metabase/driver/sql_jdbc/connection.clj
+++ b/src/metabase/driver/sql_jdbc/connection.clj
@@ -5,6 +5,7 @@
[clojure.java.jdbc :as jdbc]
[metabase.config :as config]
[metabase.connection-pool :as connection-pool]
+ [metabase.db.connection :as mdb.connection]
[metabase.driver :as driver]
[metabase.models.database :refer [Database]]
[metabase.models.interface :as mi]
@@ -211,6 +212,7 @@
"Return a JDBC connection spec that includes a cp30 `ComboPooledDataSource`. These connection pools are cached so we
don't create multiple ones for the same DB."
[db-or-id-or-spec]
+
(cond
;; db-or-id-or-spec is a Database instance or an integer ID
(u/id db-or-id-or-spec)
@@ -218,14 +220,18 @@
;; we need the Database instance no matter what (in order to compare details hash with cached value)
db (or (when (mi/instance-of? Database db-or-id-or-spec)
db-or-id-or-spec) ; passed in
- (t2/select-one [Database :id :engine :details] :id database-id) ; look up by ID
+ (t2/select-one [Database :id :engine :details :is_audit] :id database-id) ; look up by ID
(throw (ex-info (tru "Database {0} does not exist." database-id)
- {:status-code 404
- :type qp.error-type/invalid-query
- :database-id database-id})))
+ {:status-code 404
+ :type qp.error-type/invalid-query
+ :database-id database-id})))
get-fn (fn [db-id log-invalidation?]
(when-let [details (get @database-id->connection-pool db-id)]
(cond
+ ;; For the audit db, we pass the datasource for the app-db. This lets us use fewer db
+ ;; connections with the *application-db*, and 1 less connection pool.
+ (:is_audit db)
+ {:datasource (mdb.connection/data-source)}
;; details hash changed from what is cached; invalid
(let [curr-hash (get @database-id->jdbc-spec-hash db-id)
new-hash (jdbc-spec-hash db)]
diff --git a/src/metabase/query_processor/store.clj b/src/metabase/query_processor/store.clj
index 8ee4217ec9994f283c3b2fa314c063c77324d871..7a37c66e61448461ea184b32534144da888f0f37 100644
--- a/src/metabase/query_processor/store.clj
+++ b/src/metabase/query_processor/store.clj
@@ -59,7 +59,8 @@
:name
:dbms_version
:details
- :settings])
+ :settings
+ :is_audit])
(def ^:private DatabaseInstanceWithRequiredStoreKeys
(s/both
diff --git a/test/metabase/api/table_test.clj b/test/metabase/api/table_test.clj
index 2dcc00d5b2841603aaffec6c793e1cde686ab8ac..cb5d88e697c55306e5c626eb2104c99ce2661a66 100644
--- a/test/metabase/api/table_test.clj
+++ b/test/metabase/api/table_test.clj
@@ -49,7 +49,8 @@
:refingerprint nil
:auto_run_queries true
:settings nil
- :cache_ttl nil}))
+ :cache_ttl nil
+ :is_audit false}))
(defn- table-defaults []
(merge