Skip to content
Snippets Groups Projects
Unverified Commit 4cbbc961 authored by bryan's avatar bryan Committed by GitHub
Browse files

Autoload instance analytics content (when EE and avaliable) (#31314)

* wip

* fix caching behavior for audit db in database-id->connection-pool

* test that caching behavior works as advertised

- db->pooled-connection-spec always returns the same pool for audit-db
- audit-db-id is not in the database-id->connection-pool cache

* DISABLE audit db exposure through GET api/database

* Lots of things

- automagically loads instance analytics on app startup
- TODO: logging in after that is bokred

* add some logs / fix comment

* remove internal_analytics resource

* fix linter issues + update logging

* add tests for existing and missing content

- add test replica of content into test-resources

* remove unused require

* delete sample db + personal collection from test-resources

* get instance_analytics resource off the resource path

* point it to the proper IA resource

* un-hotwire the change to show audit DB on GET /databases

* fix test + lookup the right resource

* remove creator_id references from instance_analytics export test data

- get test feedback

* Revert "remove creator_id references from instance_analytics export test data"

This reverts commit 1aa9d4b1dad304ac0c6d88221f81432d78fc88f0.

* fix tests

* lint

* fix tests

* more test fixing

* linter fix

* revert yarn.lock

* Look up the correct instance_analytics resource

* cleanup comment in serialization/cmd.clj

* silence noisy serialization import logs

* remove a bunch of unused serialization files

* rename Audit Database + docstring

* rename audit database everywhere

* Put the database and tables back

* constrain tests to postgres

* ignore ia data in a test
parent 0c9ef79d
No related branches found
No related tags found
No related merge requests found
Showing
with 515 additions and 33 deletions
(ns metabase-enterprise.audit-db
(:require [metabase.config :as config]
(:require [clojure.java.io :as io]
[metabase-enterprise.serialization.cmd :as serialization.cmd]
[metabase.config :as config]
[metabase.db.env :as mdb.env]
[metabase.models.database :refer [Database]]
[metabase.public-settings.premium-features :refer [defenterprise]]
......@@ -27,7 +29,7 @@
(t2/delete! :permissions {:where [:like :object (str "%/db/" id "/%")]})
(t2/insert! Database {:is_audit true
:id default-audit-db-id
:name "Audit Database"
:name "Internal Metabase Database"
:description "Internal Audit DB used to power metabase analytics."
:engine engine
:is_full_sync true
......@@ -44,18 +46,22 @@
(cond
(nil? audit-db)
(u/prog1 ::installed
(log/info "Audit DB does not exist, Installing...")
(log/info "Installing Audit DB...")
(install-database! mdb.env/db-type))
(not= mdb.env/db-type (:engine audit-db))
(u/prog1 ::updated
(log/infof "Updating the Audit DB engine to %s." (name mdb.env/db-type))
(log/infof "App DB change detected. Changing Audit DB source to match: %s." (name mdb.env/db-type))
(t2/update! Database :is_audit true {:engine mdb.env/db-type})
(ensure-db-installed!))
:else
::no-op)))
(def analytics-root-dir-resource
"Where to look for analytics content created by Metabase to load into the app instance on startup."
(io/resource "instance_analytics"))
(defenterprise ensure-audit-db-installed!
"EE implementation of `ensure-db-installed!`. Also forces an immediate sync on audit-db."
:feature :none
......@@ -63,7 +69,15 @@
(u/prog1 (ensure-db-installed!)
;; There's a sync scheduled, but we want to force a sync right away:
(if-let [audit-db (t2/select-one :model/Database {:where [:= :is_audit true]})]
(do (log/info "Audit DB installed, beginning Audit DB Sync...")
(sync-metadata/sync-db-metadata! audit-db))
(do (log/info "Beginning Audit DB Sync...")
(log/with-no-logs (sync-metadata/sync-db-metadata! audit-db))
(log/info "Audit DB Sync Complete."))
(when (not config/is-prod?)
(log/warn "Audit DB was not installed correctly!!")))))
(log/warn "Audit DB was not installed correctly!!")))
;; load instance analytics content (collections/dashboards/cards/etc.) when the resource exists:
(when analytics-root-dir-resource
(log/info (str "Loading Analytics Content from: " analytics-root-dir-resource))
(let [report (log/with-no-logs (serialization.cmd/v2-load analytics-root-dir-resource {}))]
(if (not-empty (:errors report))
(log/info (str "Error Loading Analytics Content: " (pr-str report)))
(log/info (str "Loading Analytics Content Complete (" (count (:seen report)) ") entities synchronized.")))))))
......@@ -25,6 +25,7 @@
[metabase.util :as u]
[metabase.util.i18n :refer [deferred-trs trs]]
[metabase.util.log :as log]
[metabase.util.malli :as mu]
[metabase.util.schema :as su]
[schema.core :as s]
[toucan2.core :as t2]))
......@@ -71,11 +72,12 @@
(log/error e (trs "ERROR LOAD from {0}: {1}" path (.getMessage e)))
(throw e)))))
(defn v2-load
(mu/defn v2-load
"SerDes v2 load entry point.
opts are passed to load-metabase"
[path opts]
[path
opts :- [:map [:abort-on-error {:optional true} [:maybe :boolean]]]]
(plugins/load-plugins!)
(mdb/setup-db!)
; TODO This should be restored, but there's no manifest or other meta file written by v2 dumps.
......@@ -181,17 +183,18 @@
(defn v2-dump
"Exports Metabase app data to directory at path"
[path {:keys [user-email collections] :as opts}]
[path {:keys [user-email collection-ids] :as opts}]
(log/info (trs "Exporting Metabase to {0}" path) (u/emoji "🏭 🚛💨"))
(mdb/setup-db!)
(t2/select User) ;; TODO -- why??? [editor's note: this comment originally from Cam]
(serdes/with-cache
(-> (cond-> opts
(seq collections) (assoc :targets (v2.extract/make-targets-of-type "Collection" collections))
user-email (assoc :user-id (t2/select-one-pk User :email user-email :is_superuser true)))
(seq collection-ids) (assoc :targets (v2.extract/make-targets-of-type "Collection" collection-ids))
user-email (assoc :user-id (t2/select-one-pk User :email user-email :is_superuser true)))
v2.extract/extract
(v2.storage/store! path)))
(log/info (trs "Export to {0} complete!" path) (u/emoji "🚛💨 📦")))
(log/info (trs "Export to {0} complete!" path) (u/emoji "🚛💨 📦"))
::v2-dump-complete)
(defn seed-entity-ids
"Add entity IDs for instances of serializable models that don't already have them.
......
......@@ -41,7 +41,7 @@
(defn make-targets-of-type
"Returns a targets seq with model type and given ids"
[model-name ids]
(for [id ids] [model-name id]))
(mapv vector (repeat model-name) ids))
(defn- collection-set-for-user
"Returns a set of collection IDs to export for the provided user, if any.
......
(ns metabase-enterprise.audit-db-test
(:require [clojure.test :refer [deftest is]]
(:require [clojure.java.io :as io]
[clojure.string :as str]
[clojure.test :refer [deftest is]]
[metabase-enterprise.audit-db :as audit-db]
[metabase.models.database :refer [Database]]
[metabase.test :as mt]
[toucan2.core :as t2]))
(defmacro with-audit-db-restoration [& body]
`(let [original-audit-db# (t2/select-one Database :is_audit true)]
(try
(t2/delete! Database :is_audit true)
~@body
(finally
(t2/delete! Database :is_audit true)
(when original-audit-db#
(#'metabase.core/ensure-audit-db-installed!))))))
(deftest audit-db-is-installed-then-left-alone
(let [original-audit-db (t2/select-one Database :is_audit true)]
(try
(mt/test-drivers #{:postgres}
(with-audit-db-restoration
(t2/delete! Database :is_audit true)
(is (= :metabase-enterprise.audit-db/installed (audit-db/ensure-db-installed!)))
(is (= :metabase-enterprise.audit-db/no-op (audit-db/ensure-db-installed!)))
(t2/update! Database :is_audit true {:engine "datomic"})
(is (= :metabase-enterprise.audit-db/updated (audit-db/ensure-db-installed!)))
(is (= :metabase-enterprise.audit-db/no-op (audit-db/ensure-db-installed!)))
(is (= :metabase-enterprise.audit-db/no-op (audit-db/ensure-db-installed!))))))
(deftest audit-db-content-is-not-installed-when-not-found
(mt/test-drivers #{:postgres}
(with-audit-db-restoration
(with-redefs [audit-db/analytics-root-dir-resource nil]
(is (= nil audit-db/analytics-root-dir-resource))
(is (= :metabase-enterprise.audit-db/installed (audit-db/ensure-audit-db-installed!)))
(is (= 13371337 (t2/select-one-fn :id 'Database {:where [:= :is_audit true]}))
"Audit DB is installed.")
(is (= [] (t2/select 'Card {:where [:= :database_id 13371337]}))
"No cards created for Audit DB.")))))
(finally
(t2/delete! Database :is_audit true)
(when original-audit-db
(audit-db/ensure-audit-db-installed!))))))
(deftest audit-db-content-is-installed-when-found
(mt/test-drivers #{:postgres}
(with-audit-db-restoration
(with-redefs [audit-db/analytics-root-dir-resource (io/resource "instance_analytics_skip")]
(is (str/ends-with? (str audit-db/analytics-root-dir-resource)
"instance_analytics_skip"))
(is (= :metabase-enterprise.audit-db/installed (audit-db/ensure-audit-db-installed!)))
(is (= 13371337 (t2/select-one-fn :id 'Database {:where [:= :is_audit true]}))
"Audit DB is installed.")
(is (not= 0 (t2/count 'Card {:where [:= :database_id 13371337]}))
"Cards should be created for Audit DB when the content is there.")))))
(ns metabase-enterprise.serialization.api.serialize-test
(:require
[clojure.string :as str]
[clojure.test :refer :all]
[metabase.models :refer [Card Collection Dashboard DashboardCard]]
[metabase.public-settings.premium-features-test
......@@ -51,7 +52,11 @@
(count (files "collections" collection-filename "cards")))))
(testing "collections"
(is (= 1
(count (remove #{"cards" "dashboards" "timelines"} (files "collections"))))))
(->> (files "collections")
(remove #{"cards" "dashboards" "timelines"})
;; TODO: use better IA test data
(remove #(str/ends-with? % "instance_analytics"))
count))))
(testing "dashboards"
(is (= 1
(count (files "collections" collection-filename "dashboards")))))))))))
......
......@@ -231,9 +231,11 @@
include-saved-questions-db?
include-saved-questions-tables?
include-editable-data-model?
include-analytics?
exclude-uneditable-details?]}]
(let [dbs (t2/select Database {:where [:= :is_audit false]
:order-by [:%lower.name :%lower.engine]})
(let [dbs (t2/select Database (merge {:order-by [:%lower.name :%lower.engine]}
(when-not include-analytics?
{:where [:= :is_audit false]})))
filter-by-data-access? (not (or include-editable-data-model? exclude-uneditable-details?))]
(cond-> (add-native-perms-info dbs)
include-tables? add-tables
......@@ -262,12 +264,14 @@
* `exclude_uneditable_details` will only include DBs for which the current user can edit the DB details. Has no
effect unless Enterprise Edition code is available and the advanced-permissions feature is enabled."
[include_tables include_cards include saved include_editable_data_model exclude_uneditable_details]
[include_tables include_cards include saved include_editable_data_model exclude_uneditable_details
include_analytics]
{include_tables [:maybe :boolean]
include_cards [:maybe :boolean]
include (mu/with-api-error-message
[:maybe [:= "tables"]]
(deferred-tru "include must be either empty or the value 'tables'"))
include_analytics [:maybe :boolean]
saved [:maybe :boolean]
include_editable_data_model [:maybe :boolean]
exclude_uneditable_details [:maybe :boolean]}
......@@ -284,7 +288,8 @@
:include-saved-questions-db? include-saved-questions-db?
:include-saved-questions-tables? include-saved-questions-tables?
:include-editable-data-model? include_editable_data_model
:exclude-uneditable-details? exclude_uneditable_details)
:exclude-uneditable-details? exclude_uneditable_details
:include-analytics? include_analytics)
[])]
{:data db-list-res
:total (count db-list-res)}))
......
......@@ -191,3 +191,9 @@
(macros/case
:cljs (glogi-spy (str *ns*) level expr #(format ~fmt %))
:clj `(spyf ~level ~fmt ~expr))))
(defmacro with-no-logs
"Turns off logs in body."
[& body]
`(binding [clojure.tools.logging/*logger-factory* clojure.tools.logging.impl/disabled-logger-factory]
~@body))
......@@ -126,18 +126,20 @@
:authority_level "official"}
Collection _ {:name "Crowberto's Child Collection"
:location (collection/location-path crowberto-root)}]
(let [public-collections #{"Our analytics" (:name collection) "Collection with Items" "subcollection"}
(let [public-collections #{"Our analytics" (:name collection) "Collection with Items" "subcollection" }
crowbertos (set (map :name (mt/user-http-request :crowberto :get 200 "collection")))
crowbertos-with-excludes (set (map :name (mt/user-http-request :crowberto :get 200 "collection" :exclude-other-user-collections true)))
luckys (set (map :name (mt/user-http-request :lucky :get 200 "collection")))]
(is (= (into (set (map :name (t2/select Collection))) public-collections)
crowbertos))
luckys (set (map :name (mt/user-http-request :lucky :get 200 "collection")))
;; TODO better IA test data
hide-ia-user #(set (remove #{"Instance Analytics" "Audit" "a@a.a a@a.a's Personal Collection" "a@a.a's Personal Collection"} %))]
(is (= (hide-ia-user (into (set (map :name (t2/select Collection))) public-collections))
(hide-ia-user crowbertos)))
(is (= (into public-collections #{"Crowberto Corv's Personal Collection" "Crowberto's Child Collection"})
crowbertos-with-excludes))
(hide-ia-user crowbertos-with-excludes)))
(is (true? (contains? crowbertos "Lucky Pigeon's Personal Collection")))
(is (false? (contains? crowbertos-with-excludes "Lucky Pigeon's Personal Collection")))
(is (= (conj public-collections (:name collection) "Lucky Pigeon's Personal Collection")
luckys))
(hide-ia-user luckys)))
(is (false? (contains? luckys "Crowberto Corv's Personal Collection")))))))
(testing "Personal Collection's name and slug should be returned in user's locale"
......
authority_level: null
description: null
archived: false
slug: a_a_a_a_a_a_s_personal_collection
color: '#31698A'
name: a@a.a a@a.a's Personal Collection
personal_owner_id: a@a.a
type: null
parent_id: null
serdes/meta:
- model: Collection
id: Lq6nt9JfsdTZXXz6cj1mX
label: a_a_a_a_a_a_s_personal_collection
entity_id: Lq6nt9JfsdTZXXz6cj1mX
namespace: null
created_at: '2023-06-07T23:04:57.548225Z'
description: null
archived: false
collection_position: null
table_id:
- Internal Metabase Database
- public
- core_user
result_metadata: null
database_id: Internal Metabase Database
enable_embedding: false
collection_id: null
query_type: query
name: Core User
creator_id: a@a.a
made_public_by_id: null
embedding_params: null
cache_ttl: null
dataset_query:
database: Internal Metabase Database
type: query
query:
source-table:
- Internal Metabase Database
- public
- core_user
parameter_mappings: []
serdes/meta:
- model: Card
id: 3E5v6qwg_gPIbEGKMduFi
label: core_user
display: table
entity_id: 3E5v6qwg_gPIbEGKMduFi
collection_preview: true
visualization_settings:
table.pivot_column: id
table.cell_column: reset_triggered
column_settings: null
parameters: []
dataset: false
created_at: '2023-06-07T23:06:31.770954Z'
public_uuid: null
description: null
archived: false
collection_position: null
ordered_cards:
- size_x: 15
dashboard_tab_id: null
action_id: null
col: 0
parameter_mappings: []
card_id: 3E5v6qwg_gPIbEGKMduFi
entity_id: ifEJXWx0-orl38ouNpMJC
visualization_settings:
column_settings: null
size_y: 7
created_at: '2023-06-07T23:06:38.669473Z'
row: 0
enable_embedding: false
collection_id: tYhkn_-pFTWcCTSwhgCBP
show_in_getting_started: false
name: IA Dash
caveats: null
creator_id: a@a.a
made_public_by_id: null
embedding_params: null
cache_ttl: null
ordered_tabs: []
serdes/meta:
- model: Dashboard
id: WFmN9gRmW1LCfzHneB4QH
label: ia_dash
position: null
entity_id: WFmN9gRmW1LCfzHneB4QH
parameters: []
auto_apply_filters: true
created_at: '2023-06-07T23:05:41.204825Z'
public_uuid: null
points_of_interest: null
authority_level: null
description: null
archived: false
slug: instance_analytics
color: '#509EE3'
name: Instance Analytics
personal_owner_id: null
type: null
parent_id: null
serdes/meta:
- model: Collection
id: tYhkn_-pFTWcCTSwhgCBP
label: instance_analytics
entity_id: tYhkn_-pFTWcCTSwhgCBP
namespace: null
created_at: '2023-06-07T23:05:08.768176Z'
description: Internal Audit DB used to power metabase analytics.
cache_field_values_schedule: 0 0 19 * * ? *
timezone: UTC
auto_run_queries: true
metadata_sync_schedule: 0 56 * * * ? *
name: Internal Metabase Database
settings: null
caveats: null
creator_id: null
is_full_sync: true
cache_ttl: null
is_sample: false
is_on_demand: false
serdes/meta:
- model: Database
id: Internal Metabase Database
options: null
engine: postgres
initial_sync_status: complete
is_audit: true
dbms_version:
flavor: PostgreSQL
version: '14.2'
semantic-version:
- 14
- 2
refingerprint: null
created_at: '2023-06-07T23:03:21.08968Z'
points_of_interest: null
description: An action is something you can do, such as run a readwrite query
entity_type: null
schema: public
show_in_getting_started: false
name: action
caveats: null
active: true
db_id: Internal Metabase Database
serdes/meta:
- model: Database
id: Internal Metabase Database
- model: Schema
id: public
- model: Table
id: action
visibility_type: null
field_order: database
initial_sync_status: complete
display_name: Action
created_at: '2023-06-07T23:03:21.720428Z'
points_of_interest: null
description: Whether or not the action has been archived
database_type: bool
semantic_type: null
table_id:
- Internal Metabase Database
- public
- action
coercion_strategy: null
name: archived
fingerprint_version: 0
has_field_values: null
settings: null
caveats: null
fk_target_field_id: null
dimensions: []
custom_position: 0
effective_type: type/Boolean
active: true
nfc_path: null
parent_id: null
last_analyzed: null
serdes/meta:
- model: Database
id: Internal Metabase Database
- model: Schema
id: public
- model: Table
id: action
- model: Field
id: archived
database_is_auto_increment: false
json_unfolding: false
position: 13
visibility_type: normal
preview_display: true
display_name: Archived
database_position: 13
database_required: false
fingerprint: null
created_at: '2023-06-07T23:03:22.278337Z'
base_type: type/Boolean
points_of_interest: null
description: The timestamp of when the action was created
database_type: timestamptz
semantic_type: null
table_id:
- Internal Metabase Database
- public
- action
coercion_strategy: null
name: created_at
fingerprint_version: 0
has_field_values: null
settings: null
caveats: null
fk_target_field_id: null
dimensions: []
custom_position: 0
effective_type: type/DateTimeWithLocalTZ
active: true
nfc_path: null
parent_id: null
last_analyzed: null
serdes/meta:
- model: Database
id: Internal Metabase Database
- model: Schema
id: public
- model: Table
id: action
- model: Field
id: created_at
database_is_auto_increment: false
json_unfolding: false
position: 1
visibility_type: normal
preview_display: true
display_name: Created At
database_position: 1
database_required: false
fingerprint: null
created_at: '2023-06-07T23:03:22.278337Z'
base_type: type/DateTimeWithLocalTZ
points_of_interest: null
description: The user who created the action
database_type: int4
semantic_type: type/FK
table_id:
- Internal Metabase Database
- public
- action
coercion_strategy: null
name: creator_id
fingerprint_version: 0
has_field_values: null
settings: null
caveats: null
fk_target_field_id:
- Internal Metabase Database
- public
- core_user
- id
dimensions: []
custom_position: 0
effective_type: type/Integer
active: true
nfc_path: null
parent_id: null
last_analyzed: null
serdes/meta:
- model: Database
id: Internal Metabase Database
- model: Schema
id: public
- model: Table
id: action
- model: Field
id: creator_id
database_is_auto_increment: false
json_unfolding: false
position: 12
visibility_type: normal
preview_display: true
display_name: Creator ID
database_position: 12
database_required: false
fingerprint: null
created_at: '2023-06-07T23:03:22.278337Z'
base_type: type/Integer
points_of_interest: null
description: The description of the action
database_type: text
semantic_type: null
table_id:
- Internal Metabase Database
- public
- action
coercion_strategy: null
name: description
fingerprint_version: 0
has_field_values: null
settings: null
caveats: null
fk_target_field_id: null
dimensions: []
custom_position: 0
effective_type: type/Text
active: true
nfc_path: null
parent_id: null
last_analyzed: null
serdes/meta:
- model: Database
id: Internal Metabase Database
- model: Schema
id: public
- model: Table
id: action
- model: Field
id: description
database_is_auto_increment: false
json_unfolding: false
position: 6
visibility_type: normal
preview_display: true
display_name: Description
database_position: 6
database_required: false
fingerprint: null
created_at: '2023-06-07T23:03:22.278337Z'
base_type: type/Text
points_of_interest: null
description: Random NanoID tag for unique identity.
database_type: bpchar
semantic_type: null
table_id:
- Internal Metabase Database
- public
- action
coercion_strategy: null
name: entity_id
fingerprint_version: 0
has_field_values: null
settings: null
caveats: null
fk_target_field_id: null
dimensions: []
custom_position: 0
effective_type: type/Text
active: true
nfc_path: null
parent_id: null
last_analyzed: null
serdes/meta:
- model: Database
id: Internal Metabase Database
- model: Schema
id: public
- model: Table
id: action
- model: Field
id: entity_id
database_is_auto_increment: false
json_unfolding: false
position: 14
visibility_type: normal
preview_display: true
display_name: Entity ID
database_position: 14
database_required: false
fingerprint: null
created_at: '2023-06-07T23:03:22.278337Z'
base_type: type/Text
points_of_interest: null
description: null
database_type: int4
semantic_type: type/PK
table_id:
- Internal Metabase Database
- public
- action
coercion_strategy: null
name: id
fingerprint_version: 0
has_field_values: null
settings: null
caveats: null
fk_target_field_id: null
dimensions: []
custom_position: 0
effective_type: type/Integer
active: true
nfc_path: null
parent_id: null
last_analyzed: null
serdes/meta:
- model: Database
id: Internal Metabase Database
- model: Schema
id: public
- model: Table
id: action
- model: Field
id: id
database_is_auto_increment: true
json_unfolding: false
position: 0
visibility_type: normal
preview_display: true
display_name: ID
database_position: 0
database_required: false
fingerprint: null
created_at: '2023-06-07T23:03:22.278337Z'
base_type: type/Integer
points_of_interest: null
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment