Apply schema inclusion/exclusion filtering to sql-jdbc drivers (#19651)

* Apply schema inclusion/exclusion filtering to sql-jdbc drivers

Update `sql-jdbc` namespaces to handle schema inclusion/exclusion patterns when filtering schemas

Add new generic schema inclusion/exclusion test for sql-jdbc drivers that define the property

Update Snowflake and Redshift driver manifests to include schema filtering property

Create `db-details->schema-filter-patterns` util fn to turn DB details into the inclusion/exclusion patterns

Move schema inclusion/exclusion filtering code to new namespace (since it's not strictly used by `:sql-jdbc` derived drivers)

Move existing tests accordingly

Add schema inclusion/exclusion check to the new `filtered-syncable-schemas` multimethod (and updating docstring)

Change `:redshift` impl of `filtered-syncable-schemas` to call the `:sql-jdbc` version instead

Use new multimethod instead for `filtered-syncable-schemas`, and have default impl of `syncable-schemas` call that

Mark `syncable-schemas` as deprecated and include notes on the new method (and update driver markdown file accordingly)

docs/developers-guide/driver-changelog.md

@@ -47,6 +47,10 @@ If you were manipulating Field or Table aliases, we consolidated a lot of overla
   used consistently across the SQL QP code. If you need to transform generated Field aliases for any reason (such as
   escaping disallowed characters), implement this method.
 
+- `metabase.driver.sql-jdbc.sync.interface/filtered-syncable-schemas` has been added, and will eventually replace
+  `metabase.driver.sql-jdbc.sync.interface/syncable-schemas`.  It serves a similar purpose, except that it's also
+  passed the inclusion and exclusion patterns (ex: `auth*,data*`) to further filter schemas that will be synced.
+
 ### Deprecated methods and vars
 
 The following methods and vars are slated for removal in Metabase 0.45.0 unless otherwise noted.
@@ -84,6 +88,10 @@ The following methods and vars are slated for removal in Metabase 0.45.0 unless
   give drivers a chance to escape automatically generated aliases for joined Fields. This is no longer necessary,
   because `metabase.driver/escape-alias` is called on automatically generates aliases. Implement
   `metabase.driver/escape-alias` if you need to do something special.
+- `metabase.driver.sql-jdbc.sync.interface/syncable-schemas` has been deprecated in favor of
+  `metabase.driver.sql-jdbc.sync.interface/filtered-syncable-schemas` (see above). The existing default implementation
+  of `syncable-schemas` currently calls `filtered-syncable-schemas` (with `nil` filters, i.e. the filtering operation
+  is actually a no-op), but it will eventually be removed.
 
 #### Removed Methods and Vars
 

modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj

@@ -8,8 +8,8 @@
             [metabase.driver.bigquery-cloud-sdk.common :as bigquery.common]
             [metabase.driver.bigquery-cloud-sdk.params :as bigquery.params]
             [metabase.driver.bigquery-cloud-sdk.query-processor :as bigquery.qp]
-            [metabase.driver.sql-jdbc.sync.describe-database :as describe-database]
-            [metabase.models :refer [Database Table] :rename {Table MetabaseTable}] ; Table clashes with the class name below
+            [metabase.driver.sync :as driver.s]
+            [metabase.models :refer [Database Table] :rename {Table MetabaseTable}] ; Table clashes with the class below
             [metabase.query-processor.context :as context]
             [metabase.query-processor.error-type :as error-type]
             [metabase.query-processor.store :as qp.store]
@@ -67,9 +67,9 @@
           exclusion-patterns (when (= "exclusion" filter-type) filter-patterns)]
       (apply concat (for [^Dataset dataset (.iterateAll datasets)
                           :let [^DatasetId dataset-id (.. dataset getDatasetId)]
-                          :when (describe-database/include-schema? (.getDataset dataset-id)
-                                                                   inclusion-patterns
-                                                                   exclusion-patterns)]
+                          :when (driver.s/include-schema? inclusion-patterns
+                                                          exclusion-patterns
+                                                          (.getDataset dataset-id))]
                       (-> (.listTables client dataset-id (u/varargs BigQuery$TableListOption))
                           .iterateAll
                           .iterator

modules/drivers/redshift/resources/metabase-plugin.yaml

@@ -18,6 +18,9 @@ driver:
         - dbname
         - name: db
           placeholder: birds_of_the_world
+    - name: schema-filters
+      type: schema-filters
+      display-name: Schemas
     - user
     - password
     - cloud-ip-address-info

modules/drivers/redshift/src/metabase/driver/redshift.clj

@@ -11,7 +11,6 @@
             [metabase.driver.sql-jdbc.execute :as sql-jdbc.execute]
             [metabase.driver.sql-jdbc.execute.legacy-impl :as legacy]
             [metabase.driver.sql-jdbc.sync :as sql-jdbc.sync]
-            [metabase.driver.sql-jdbc.sync.describe-database :as sync.describe-database]
             [metabase.driver.sql.query-processor :as sql.qp]
             [metabase.mbql.util :as mbql.u]
             [metabase.public-settings :as pubset]
@@ -271,10 +270,11 @@
                         false))))
           reducible))))))
 
-(defmethod sql-jdbc.sync/syncable-schemas :redshift
-  [driver conn metadata]
-  (reducible-schemas-with-usage-permissions
-   conn
-   (eduction
-    (remove (set (sql-jdbc.sync/excluded-schemas driver)))
-    (sync.describe-database/all-schemas metadata))))
+(defmethod sql-jdbc.sync/filtered-syncable-schemas :redshift
+  [driver conn metadata schema-inclusion-patterns schema-exclusion-patterns]
+  (let [parent-method (get-method sql-jdbc.sync/filtered-syncable-schemas :sql-jdbc)]
+    (reducible-schemas-with-usage-permissions conn (parent-method driver
+                                                                  conn
+                                                                  metadata
+                                                                  schema-inclusion-patterns
+                                                                  schema-exclusion-patterns))))

modules/drivers/redshift/test/metabase/driver/redshift_test.clj

@@ -207,7 +207,7 @@
                     (partial into {})
                     (db/select [Field :name :database_type :base_type] :table_id table-id {:order-by [:name]}))))))))))
 
-(deftest syncable-schemas-test
+(deftest filtered-syncable-schemas-test
   (mt/test-driver :redshift
     (testing "Should filter out schemas for which the user has no perms"
       ;; create a random username and random schema name, and grant the user USAGE permission for it
@@ -224,13 +224,17 @@
                                 random-schema
                                 temp-username)
         (try
-          (binding [redshift.test/*use-original-syncable-schemas-impl?* true]
+          (binding [redshift.test/*use-original-filtered-syncable-schemas-impl?* true]
             (mt/with-temp Database [db {:engine :redshift, :details (assoc db-det :user temp-username :password user-pw)}]
               (with-open [conn (jdbc/get-connection (sql-jdbc.conn/db->pooled-connection-spec db))]
                 (let [schemas (reduce conj
                                       #{}
-                                      (sql-jdbc.sync/syncable-schemas :redshift conn (.getMetaData conn)))]
-                  (testing "syncable-schemas for the user should contain the newly created random schema"
+                                      (sql-jdbc.sync/filtered-syncable-schemas :redshift
+                                                                               conn
+                                                                               (.getMetaData conn)
+                                                                               nil
+                                                                               nil))]
+                  (testing "filtered-syncable-schemas for the user should contain the newly created random schema"
                     (is (contains? schemas random-schema)))
                   (testing "should not contain the current session-schema name (since that was never granted)"
                     (is (not (contains? schemas redshift.test/session-schema-name))))))))
@@ -245,7 +249,7 @@
 
     (testing "Should filter out non-existent schemas (for which nobody has permissions)"
       (let [fake-schema-name (u/qualified-name ::fake-schema)]
-        (binding [redshift.test/*use-original-syncable-schemas-impl?* true]
+        (binding [redshift.test/*use-original-filtered-syncable-schemas-impl?* true]
           ;; override `all-schemas` so it returns our fake schema in addition to the real ones.
           (with-redefs [sync.describe-database/all-schemas (let [orig sync.describe-database/all-schemas]
                                                              (fn [metadata]
@@ -258,7 +262,7 @@
                           (reduce
                            conj
                            #{}
-                           (sql-jdbc.sync/syncable-schemas :redshift conn (.getMetaData conn))))]
+                           (sql-jdbc.sync/filtered-syncable-schemas :redshift conn (.getMetaData conn) nil nil)))]
                   (testing "if schemas-with-usage-permissions is disabled, the ::fake-schema should come back"
                     (with-redefs [redshift/reducible-schemas-with-usage-permissions (fn [_ reducible]
                                                                                       reducible)]

modules/drivers/redshift/test/metabase/test/data/redshift.clj

@@ -82,18 +82,19 @@
   [_]
   (execute! "DROP SCHEMA IF EXISTS %s CASCADE; CREATE SCHEMA %s;" session-schema-name session-schema-name))
 
-(defonce ^:private ^{:arglists '([driver connection metadata])} original-syncable-schemas
-  (get-method sql-jdbc.sync/syncable-schemas :redshift))
+(defonce ^:private ^{:arglists '([driver connection metadata _ _])}
+  original-filtered-syncable-schemas
+  (get-method sql-jdbc.sync/filtered-syncable-schemas :redshift))
 
-(def ^:dynamic *use-original-syncable-schemas-impl?*
-  "Whether to use the actual prod impl for `syncable-schemas` rather than the special test one that only syncs the test
-  schema."
+(def ^:dynamic *use-original-filtered-syncable-schemas-impl?*
+  "Whether to use the actual prod impl for `filtered-syncable-schemas` rather than the special test one that only syncs
+  the test schema."
   false)
 
 ;; replace the impl the `metabase.driver.redshift`. Only sync the current test schema and the external "spectrum"
 ;; schema used for a specific test.
-(defmethod sql-jdbc.sync/syncable-schemas :redshift
-  [driver conn metadata]
-  (if *use-original-syncable-schemas-impl?*
-    (original-syncable-schemas driver conn metadata)
+(defmethod sql-jdbc.sync/filtered-syncable-schemas :redshift
+  [driver conn metadata schema-inclusion-filters schema-exclusion-filters]
+  (if *use-original-filtered-syncable-schemas-impl?*
+    (original-filtered-syncable-schemas driver conn metadata schema-inclusion-filters schema-exclusion-filters)
     #{session-schema-name "spectrum"}))

modules/drivers/snowflake/resources/metabase-plugin.yaml

@@ -27,10 +27,9 @@ driver:
         - name: db
           required: true
           display-name: Database name (case sensitive)
-    - name: schema
-      display-name: Schema (optional)
-      helper-text: Only add tables to Metabase that come from a specific schema.
-      placeholder: just_crows
+    - name: schema-filters
+      type: schema-filters
+      display-name: Schemas
     - name: role
       display-name: Role (optional)
       helper-text: Specify a role to override the database user’s default role.

modules/drivers/snowflake/src/metabase/driver/snowflake.clj

@@ -17,6 +17,7 @@
             [metabase.driver.sql-jdbc.sync :as sql-jdbc.sync]
             [metabase.driver.sql.query-processor :as sql.qp]
             [metabase.driver.sql.util.unprepare :as unprepare]
+            [metabase.driver.sync :as driver.s]
             [metabase.query-processor.store :as qp.store]
             [metabase.query-processor.util.add-alias-info :as add]
             [metabase.util :as u]
@@ -239,14 +240,19 @@
         excluded-schemas (set (sql-jdbc.sync/excluded-schemas driver))]
     (qp.store/with-store
       (qp.store/fetch-and-store-database! (u/the-id database))
-      (let [spec (sql-jdbc.conn/db->pooled-connection-spec database)
-            sql  (format "SHOW OBJECTS IN DATABASE \"%s\"" db-name)]
+      (let [spec            (sql-jdbc.conn/db->pooled-connection-spec database)
+            sql             (format "SHOW OBJECTS IN DATABASE \"%s\"" db-name)
+            schema-patterns (driver.s/db-details->schema-filter-patterns "schema-filters" database)
+            [inclusion-patterns exclusion-patterns] schema-patterns]
         (log/tracef "[Snowflake] %s" sql)
         (with-open [conn (jdbc/get-connection spec)]
           {:tables (into
                     #{}
                     (comp (filter (fn [{schema :schema_name, table-name :name}]
                                     (and (not (contains? excluded-schemas schema))
+                                         (driver.s/include-schema? inclusion-patterns
+                                                                   exclusion-patterns
+                                                                   schema)
                                          (sql-jdbc.sync/have-select-privilege? driver conn schema table-name))))
                           (map (fn [{schema :schema_name, table-name :name, remark :comment}]
                                  {:name        table-name

src/metabase/driver/sql_jdbc/sync.clj

@@ -15,6 +15,7 @@
   db-default-timezone
   excluded-schemas
   fallback-metadata-query
+  filtered-syncable-schemas
   have-select-privilege?
   syncable-schemas]
 

src/metabase/driver/sql_jdbc/sync/describe_database.clj

@@ -9,9 +9,11 @@
             [metabase.driver.sql-jdbc.sync.common :as common]
             [metabase.driver.sql-jdbc.sync.interface :as i]
             [metabase.driver.sql.query-processor :as sql.qp]
+            [metabase.driver.sync :as driver.s]
+            [metabase.driver.util :as driver.u]
+            [metabase.models :refer [Database]]
             [metabase.util.honeysql-extensions :as hx])
-  (:import [java.sql Connection DatabaseMetaData ResultSet]
-           java.util.regex.Pattern))
+  (:import [java.sql Connection DatabaseMetaData ResultSet]))
 
 (defmethod i/excluded-schemas :sql-jdbc [_] nil)
 
@@ -24,9 +26,10 @@
    (fn [^ResultSet rs]
      #(.getString rs "TABLE_SCHEM"))))
 
-(defmethod i/syncable-schemas :sql-jdbc
-  [driver _ metadata]
+(defmethod i/filtered-syncable-schemas :sql-jdbc
+  [driver _ metadata schema-inclusion-patterns schema-exclusion-patterns]
   (eduction (remove (set (i/excluded-schemas driver)))
+            (filter (partial driver.s/include-schema? schema-inclusion-patterns schema-exclusion-patterns))
             (all-schemas metadata)))
 
 (defn simple-select-probe-query
@@ -97,7 +100,7 @@
 
   This is as much as 15x faster for Databases with lots of system tables than `post-filtered-active-tables` (4 seconds
   vs 60)."
-  [driver ^Connection conn & [db-name-or-nil]]
+  [driver ^Connection conn & [db-name-or-nil schema-inclusion-filters schema-exclusion-filters]]
   {:pre [(instance? Connection conn)]}
   (let [metadata (.getMetaData conn)]
     (eduction
@@ -105,24 +108,35 @@
                      (db-tables driver metadata schema db-name-or-nil)))
            (filter (fn [{table-schema :schema, table-name :name}]
                      (i/have-select-privilege? driver conn table-schema table-name))))
-     (i/syncable-schemas driver conn metadata))))
+     (i/filtered-syncable-schemas driver conn metadata schema-inclusion-filters schema-exclusion-filters))))
 
 (defmethod i/active-tables :sql-jdbc
-  [driver connection]
-  (fast-active-tables driver connection))
+  [driver connection schema-inclusion-filters schema-exclusion-filters]
+  (fast-active-tables driver connection nil schema-inclusion-filters schema-exclusion-filters))
 
 (defn post-filtered-active-tables
   "Alternative implementation of `active-tables` best suited for DBs with little or no support for schemas. Fetch *all*
   Tables, then filter out ones whose schema is in `excluded-schemas` Clojure-side."
-  [driver ^Connection conn & [db-name-or-nil]]
+  [driver ^Connection conn & [db-name-or-nil schema-inclusion-filters schema-exclusion-filters]]
   {:pre [(instance? Connection conn)]}
   (eduction
    (filter (let [excluded (i/excluded-schemas driver)]
              (fn [{table-schema :schema, table-name :name}]
                (and (not (contains? excluded table-schema))
+                    (driver.s/include-schema? schema-inclusion-filters schema-exclusion-filters table-schema)
                     (i/have-select-privilege? driver conn table-schema table-name)))))
    (db-tables driver (.getMetaData conn) nil db-name-or-nil)))
 
+(defn- db-or-id-or-spec->database [db-or-id-or-spec]
+  (cond (instance? (class Database) db-or-id-or-spec)
+        db-or-id-or-spec
+
+        (int? db-or-id-or-spec)
+        (Database db-or-id-or-spec)
+
+        true
+        nil))
+
 (defn describe-database
   "Default implementation of `driver/describe-database` for SQL JDBC drivers. Uses JDBC DatabaseMetaData."
   [driver db-or-id-or-spec]
@@ -131,39 +145,15 @@
              ;; is. Not sure how much of a difference that makes since we're not running this inside a transaction,
              ;; but better safe than sorry
              (sql-jdbc.execute/set-best-transaction-level! driver conn)
-             (into #{} (i/active-tables driver conn)))})
-
-(defn- schema-pattern->re-pattern ^Pattern [schema-pattern]
-  (re-pattern (-> (str/replace schema-pattern #"(^|[^\\\\])\*" "$1.*")
-                  (str/replace #"(^|[^\\\\])," "$1|"))))
-
-(defn- schema-patterns->filter-fn*
-  [inclusion-patterns exclusion-patterns]
-  (let [inclusion-blank? (str/blank? inclusion-patterns)
-        exclusion-blank? (str/blank? exclusion-patterns)]
-    (cond
-      (and inclusion-blank? exclusion-blank?)
-      (constantly true)
-
-      (and (not inclusion-blank?) (not exclusion-blank?))
-      (throw (ex-info "Inclusion and exclusion patterns cannot both be specified"
-                      {::inclusion-patterns inclusion-patterns
-                       ::exclusion-patterns exclusion-patterns}))
-
-      true
-      (let [inclusion? exclusion-blank?
-            pattern    (schema-pattern->re-pattern (if inclusion? inclusion-patterns exclusion-patterns))]
-        (fn [s]
-          (let [m        (.matcher pattern s)
-                matches? (.matches m)]
-            (if inclusion? matches? (not matches?))))))))
-
-(def ^:private schema-patterns->filter-fn (memoize schema-patterns->filter-fn*))
-
-(defn include-schema?
-  ;; TODO: add more docstring details here, and move to different ns (not strictly JDBC)
-  "Returns true of the given `schema-name` should be included/synced, considering the given `inclusion-patterns` and
-  `exclusion-patterns`."
-  [schema-name inclusion-patterns exclusion-patterns]
-  (let [filter-fn (schema-patterns->filter-fn inclusion-patterns exclusion-patterns)]
-    (filter-fn schema-name)))
+             (let [schema-filter-prop      (driver.u/find-schema-filters-prop driver)
+                   has-schema-filter-prop? (some? schema-filter-prop)
+                   default-active-tbl-fn   #(into #{} (i/active-tables driver conn nil nil))]
+               (if has-schema-filter-prop?
+                 (if-let [database (db-or-id-or-spec->database db-or-id-or-spec)]
+                   (let [prop-nm                                 (:name schema-filter-prop)
+                         [inclusion-patterns exclusion-patterns] (driver.s/db-details->schema-filter-patterns
+                                                                  prop-nm
+                                                                  database)]
+                     (into #{} (i/active-tables driver conn inclusion-patterns exclusion-patterns)))
+                   (default-active-tbl-fn))
+                 (default-active-tbl-fn))))})

src/metabase/driver/sql_jdbc/sync/interface.clj

@@ -11,7 +11,10 @@
   functions for more details on the differences.
 
   `metabase` is an instance of `DatabaseMetaData`."
-  {:arglists '([driver ^java.sql.Connection connection])}
+  {:arglists '([driver
+                ^java.sql.Connection connection
+                ^String schema-inclusion-filters
+                ^String schema-exclusion-filters])}
   driver/dispatch-on-initialized-driver
   :hierarchy #'driver/hierarchy)
 
@@ -29,14 +32,36 @@
   driver/dispatch-on-initialized-driver
   :hierarchy #'driver/hierarchy)
 
+(defmulti filtered-syncable-schemas
+  "Return a reducible sequence of string names of schemas that should be synced for the given database. Schemas for
+  which the current DB user has no `SELECT` permissions should be filtered out. The default implementation will fetch
+  a sequence of all schema names from the JDBC database metadata and filter out any schemas in `excluded-schemas`, along
+  with any that shouldn't be included based on the given inclusion and exclusion patterns (see the
+  `metabase.driver.sync` namespace for full explanation)."
+  {:added "0.43.0", :arglists '([driver
+                                 ^java.sql.Connection connection
+                                 ^java.sql.DatabaseMetaData metadata
+                                 ^String schema-inclusion-patterns
+                                 ^String schema-exclusion-patterns])}
+  driver/dispatch-on-initialized-driver
+  :hierarchy #'driver/hierarchy)
+
 (defmulti syncable-schemas
   "Return a reducible sequence of string names of schemas that should be synced for the given database. Schemas for
   which the current DB user has no `SELECT` permissions should be filtered out. The default implementation will fetch
-  a sequence of all schema names from the JDBC database metadata and filter out any schemas in `excluded-schemas`."
-  {:added "0.39.0", :arglists '([driver ^java.sql.Connection connection ^java.sql.DatabaseMetaData metadata])}
+  a sequence of all schema names from the JDBC database metadata and filter out any schemas in `excluded-schemas`.
+
+  DEPRECATED - as of 0.43, this method is deprecated in favor of [[filtered-syncable-schemas]]."
+  {:added "0.39.0", :deprecated "0.43.0",
+   :arglists '([driver ^java.sql.Connection connection ^java.sql.DatabaseMetaData metadata])}
   driver/dispatch-on-initialized-driver
   :hierarchy #'driver/hierarchy)
 
+(defmethod syncable-schemas :default
+  [driver connection metadata]
+  ;; default impl; call the filtered multimethod will nil inclusion and exclusion patterns
+  (filtered-syncable-schemas driver connection metadata nil nil))
+
 (defmulti database-type->base-type
   "Given a native DB column type (as a keyword), return the corresponding `Field` `base-type`, which should derive from
   `:type/*`. You can use `pattern-based-database-type->base-type` in this namespace to implement this using regex

src/metabase/driver/sync.clj

+(ns metabase.driver.sync
+  "General functions and utilities for sync operations across multiple drivers."
+  (:require [clojure.string :as str])
+  (:import java.util.regex.Pattern))
+
+(defn- schema-pattern->re-pattern
+  "Converts a schema pattern, as entered in the UI, into regex pattern suitable to be passed into [[re-pattern]].  The
+  conversion that happens is from commas into pipes (disjunction), and wildcard characters (`*`) into greedy wildcard
+  matchers (`.*`).  These only occur if those characters are not preceded by a backslash, which serves as an escape
+  character for purposes of this conversion.
+
+  Examples:
+    a,b => a|b
+    test* => test.*
+    foo*,*bar => foo.*|.*bar
+    crazy\\*schema => crazy\\*schema"
+  ^Pattern [schema-pattern]
+  (re-pattern (-> (str/replace schema-pattern #"(^|[^\\\\])\*" "$1.*")
+                  (str/replace #"(^|[^\\\\])," "$1|"))))
+
+(defn- schema-patterns->filter-fn*
+  [inclusion-patterns exclusion-patterns]
+  (let [inclusion-blank? (str/blank? inclusion-patterns)
+        exclusion-blank? (str/blank? exclusion-patterns)]
+    (cond
+      (and inclusion-blank? exclusion-blank?)
+      (constantly true)
+
+      (and (not inclusion-blank?) (not exclusion-blank?))
+      (throw (ex-info "Inclusion and exclusion patterns cannot both be specified"
+                      {::inclusion-patterns inclusion-patterns
+                       ::exclusion-patterns exclusion-patterns}))
+
+      true
+      (let [inclusion? exclusion-blank?
+            pattern    (schema-pattern->re-pattern (if inclusion? inclusion-patterns exclusion-patterns))]
+        (fn [s]
+          (let [m        (.matcher pattern s)
+                matches? (.matches m)]
+            (if inclusion? matches? (not matches?))))))))
+
+(def ^:private schema-patterns->filter-fn (memoize schema-patterns->filter-fn*))
+
+(defn include-schema?
+  "Returns true of the given `schema-name` should be included/synced, considering the given `inclusion-patterns` and
+  `exclusion-patterns`. Patterns are comma-separated, and can contain wildcard characters (`*`)."
+  {:added "0.42.0"}
+  [inclusion-patterns exclusion-patterns schema-name]
+  (let [filter-fn (schema-patterns->filter-fn inclusion-patterns exclusion-patterns)]
+    (filter-fn schema-name)))
+
+(defn db-details->schema-filter-patterns
+  "Given a `prop-nm` (which is expected to be a connection property of type `:schema-filters`), and a `database`
+  instance, return a vector containing [inclusion-patterns exclusion-patterns]."
+  {:added "0.42.0"}
+  [prop-nm {db-details :details :as database}]
+  (let [schema-filter-type     (get db-details (keyword (str prop-nm "-type")))
+        schema-filter-patterns (get db-details (keyword (str prop-nm "-patterns")))
+        exclusion-type?        (= "exclusion" schema-filter-type)]
+    (if exclusion-type?
+      [nil schema-filter-patterns]
+      [schema-filter-patterns nil])))

src/metabase/driver/util.clj

@@ -197,6 +197,13 @@
       :required true}
       ]))
 
+(defn find-schema-filters-prop
+  "Finds the first property of type `:schema-filters` for the given `driver` connection properties. Returns `nil`
+  if the driver has no property of that type."
+  [driver]
+  (first (filter (fn [conn-prop]
+                   (= :schema-filters (keyword (:type conn-prop))))
+           (driver/connection-properties driver))))
 
 (defn connection-props-server->client
   "Transforms `conn-props` for the given `driver` from their server side definition into a client side definition.

test/metabase/driver/sql_jdbc/sync/describe_database_test.clj

@@ -5,14 +5,18 @@
             [metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]
             [metabase.driver.sql-jdbc.sync.describe-database :as describe-database]
             [metabase.driver.sql-jdbc.sync.interface :as i]
-            [metabase.models.table :refer [Table]]
+            [metabase.driver.util :as driver.u]
+            [metabase.models :refer [Database Table]]
             [metabase.query-processor :as qp]
             [metabase.sync :as sync]
             [metabase.test :as mt]
             [metabase.test.data.one-off-dbs :as one-off-dbs]
+            [metabase.test.fixtures :as fixtures]
+            [metabase.util :as u]
             [toucan.db :as db])
-  (:import clojure.lang.ExceptionInfo
-           java.sql.ResultSet))
+  (:import java.sql.ResultSet))
+
+(use-fixtures :once (fixtures/initialize :plugins))
 
 (deftest simple-select-probe-query-test
   (testing "simple-select-probe-query shouldn't actually return any rows"
@@ -38,7 +42,7 @@
       ;; We have to mock this to make it work with all DBs
       (with-redefs [describe-database/all-schemas (constantly #{"PUBLIC"})]
         (is (= ["CATEGORIES" "CHECKINS" "USERS" "VENUES"]
-               (->> (into [] (describe-database/fast-active-tables (or driver/*driver* :h2) conn))
+               (->> (into [] (describe-database/fast-active-tables (or driver/*driver* :h2) conn nil nil))
                     (map :name)
                     sort)))))))
 
@@ -46,7 +50,7 @@
   (let [spec (sql-jdbc.conn/db->pooled-connection-spec (mt/db))]
     (with-open [conn (jdbc/get-connection spec)]
       (is (= ["CATEGORIES" "CHECKINS" "USERS" "VENUES"]
-             (->> (into [] (describe-database/post-filtered-active-tables :h2 conn))
+             (->> (into [] (describe-database/post-filtered-active-tables :h2 conn nil nil))
                   (map :name)
                   sort))))))
 
@@ -104,24 +108,45 @@
       (is (= 0
              (describe-database-with-open-resultset-count driver/*driver* (mt/db)))))))
 
-(deftest schema-filter-test
-  (doseq [[test-kind expect-match? schema-name inclusion-filters exclusion-filters]
-          [["nil filters" true "foo" nil nil]
-           ["blank filters" true "foo" "" ""]
-           ["simple inclusion filter (include)" true "foo" "foo" ""]
-           ["simple inclusion filter (exclude)" false "bar" "foo" ""]
-           ["wildcard inclusion filter" true "foo" "f*" ""]
-           ["simple exclusion filter (include)" true "bar" "" "foo"]
-           ["simple exclusion filter (exclude)" false "foo" "" "foo"]
-           ["wildcard exclusion filter" true "foo" "" "b*"]
-           ["inclusion filter with commas and wildcards (include)" true "foo" "bar,f*,baz" ""]
-           ["inclusion filter with commas and wildcards (exclude)" false "ban" "bar,f*,baz" ""]
-           ["exclusion filter with commas and wildcards (include)" true "foo" "" "ba*,fob"]
-           ["exclusion filter with commas and wildcards (exclude)" false "foo" "" "bar,baz,fo*"]]]
-    (testing (str "include-schema? works as expected for " test-kind)
-      (is (= expect-match? (describe-database/include-schema? schema-name inclusion-filters exclusion-filters))))
-    (testing "include-schema? throws an exception if both patterns are specified"
-      (is (thrown-with-msg?
-            ExceptionInfo
-            #"Inclusion and exclusion patterns cannot both be specified"
-            (describe-database/include-schema? "whatever" "foo" "bar"))))))
+(defn- sync-and-assert-filtered-tables [database assert-table-fn]
+  (mt/with-temp Database [db-filtered database]
+    (let [sync-results (sync/sync-database! db-filtered {:scan :schema})
+          tables       (db/select Table :db_id (u/the-id db-filtered))]
+      (doseq [table tables]
+        (assert-table-fn table)))))
+
+(defn- find-schema-filters-prop [driver]
+  (first (filter (fn [conn-prop]
+                   (= :schema-filters (keyword (:type conn-prop))))
+                 (driver/connection-properties driver))))
+
+(defn- schema-filtering-drivers []
+  (set (for [driver (mt/normal-drivers)
+             :when  (driver.u/find-schema-filters-prop driver)]
+         driver)))
+
+(deftest database-schema-filtering-test
+  (mt/test-drivers (schema-filtering-drivers)
+    (let [driver             (driver.u/database->driver (mt/db))
+          schema-filter-prop (find-schema-filters-prop driver)
+          filter-type-prop   (keyword (str (:name schema-filter-prop) "-type"))
+          patterns-type-prop (keyword (str (:name schema-filter-prop) "-patterns"))]
+      (testing "Filtering connections for schemas works as expected"
+        (testing " with an inclusion filter"
+          (sync-and-assert-filtered-tables {:name    (format "Test %s DB with dataset inclusion filters" driver)
+                                            :engine  driver
+                                            :details (-> (mt/db)
+                                                         :details
+                                                         (assoc filter-type-prop "inclusion"
+                                                                patterns-type-prop "s*,v*"))}
+            (fn [{schema-name :schema}]
+              (is (contains? #{\s \v} (first schema-name))))))
+        (testing " with an exclusion filter"
+          (sync-and-assert-filtered-tables {:name    (format "Test %s DB with dataset exclusion filters" driver)
+                                            :engine  driver
+                                            :details (-> (mt/db)
+                                                         :details
+                                                         (assoc filter-type-prop "exclusion"
+                                                                patterns-type-prop "v*"))}
+            (fn [{schema-name :schema}]
+              (is (not= \v (first schema-name))))))))))

test/metabase/driver/sync_test.clj

+(ns metabase.driver.sync-test
+  (:require [clojure.test :as t]
+            [metabase.driver.sync :as driver.s])
+  (:import clojure.lang.ExceptionInfo))
+
+(t/deftest schema-filter-test
+  (doseq [[test-kind expect-match? schema-name inclusion-filters exclusion-filters]
+          [["nil filters" true "foo" nil nil]
+           ["blank filters" true "foo" "" ""]
+           ["simple inclusion filter (include)" true "foo" "foo" ""]
+           ["simple inclusion filter (exclude)" false "bar" "foo" ""]
+           ["wildcard inclusion filter" true "foo" "f*" ""]
+           ["simple exclusion filter (include)" true "bar" "" "foo"]
+           ["simple exclusion filter (exclude)" false "foo" "" "foo"]
+           ["wildcard exclusion filter" true "foo" "" "b*"]
+           ["inclusion filter with commas and wildcards (include)" true "foo" "bar,f*,baz" ""]
+           ["inclusion filter with commas and wildcards (exclude)" false "ban" "bar,f*,baz" ""]
+           ["exclusion filter with commas and wildcards (include)" true "foo" "" "ba*,fob"]
+           ["exclusion filter with commas and wildcards (exclude)" false "foo" "" "bar,baz,fo*"]]]
+    (t/testing (str "include-schema? works as expected for " test-kind)
+      (t/is (= expect-match? (driver.s/include-schema? inclusion-filters exclusion-filters schema-name))))
+    (t/testing "include-schema? throws an exception if both patterns are specified"
+      (t/is (thrown-with-msg?
+             ExceptionInfo
+             #"Inclusion and exclusion patterns cannot both be specified"
+             (driver.s/include-schema? "foo" "bar" "whatever"))))))