Merge pull request #1656 from metabase/fast-active-tables

Fast implementation of `active-tables` for SQL DBs 

src/metabase/driver/generic_sql.clj

 (ns metabase.driver.generic-sql
   (:require [clojure.core.memoize :as memoize]
             [clojure.java.jdbc :as jdbc]
+            [clojure.set :as set]
             [clojure.tools.logging :as log]
             (korma [core :as k]
                    [db :as kdb])
@@ -125,13 +126,32 @@
      (let [~binding (.getMetaData conn#)]
        ~@body)))
 
-(defn- active-tables [driver database]
+
+(defn fast-active-tables
+  "Default, fast implementation of `IDriver/active-tables` best suited for DBs with lots of system tables (like Oracle).
+   Fetch list of schemas, then for each one not in `excluded-schemas`, fetch its Tables, and combine the results.
+
+   This is as much as 15x faster for Databases with lots of system tables than `post-filtered-active-tables` (4 seconds vs 60)."
+  [driver database]
+  (with-metadata [md driver database]
+    (let [all-schemas (set (map :table_schem (jdbc/result-set-seq (.getSchemas md))))
+          schemas     (set/difference all-schemas (excluded-schemas driver))]
+      (set (for [schema     schemas
+                 table-name (mapv :table_name (jdbc/result-set-seq (.getTables md nil schema nil (into-array String ["TABLE", "VIEW"]))))]
+             {:name   table-name
+              :schema schema})))))
+
+(defn post-filtered-active-tables
+  "Alternative implementation of `IDriver/active-tables` best suited for DBs with little or no support for schemas.
+   Fetch *all* Tables, then filter out ones whose schema is in `excluded-schemas` Clojure-side."
+  [driver database]
   (with-metadata [md driver database]
     (set (for [table (filter #(not (contains? (excluded-schemas driver) (:table_schem %)))
                              (jdbc/result-set-seq (.getTables md nil nil nil (into-array String ["TABLE", "VIEW"]))))]
            {:name   (:table_name table)
             :schema (:table_schem table)}))))
 
+
 (defn- active-column-names->type [driver table]
   (with-metadata [md driver @(:db table)]
     (into {} (for [{:keys [column_name type_name]} (jdbc/result-set-seq (.getColumns md nil (:schema table) (:name table) nil))]
@@ -245,7 +265,7 @@
            'metabase.driver.generic-sql.query-processor)
   (merge driver/IDriverDefaultsMixin
          {:active-column-names->type active-column-names->type
-          :active-tables             active-tables
+          :active-tables             fast-active-tables
           :can-connect?              can-connect?
           :features                  features
           :field-avg-length          field-avg-length

src/metabase/driver/h2.clj

@@ -198,7 +198,8 @@
 (extend H2Driver
   driver/IDriver
   (merge (sql/IDriverSQLDefaultsMixin)
-         {:date-interval                     date-interval
+         {:active-tables                     sql/post-filtered-active-tables
+          :date-interval                     date-interval
           :details-fields                    (constantly [{:name         "db"
                                                            :display-name "Connection String"
                                                            :placeholder  "file:/Users/camsaul/bird_sightings/toucans;AUTO_SERVER=TRUE"

src/metabase/driver/mysql.clj

@@ -141,7 +141,8 @@
 (extend MySQLDriver
   driver/IDriver
   (merge (sql/IDriverSQLDefaultsMixin)
-         {:date-interval                     date-interval
+         {:active-tables                     sql/post-filtered-active-tables
+          :date-interval                     date-interval
           :details-fields                    (constantly [{:name         "host"
                                                            :display-name "Host"
                                                            :default      "localhost"}

src/metabase/driver/sqlite.clj

@@ -108,7 +108,8 @@
 (extend SQLiteDriver
   driver/IDriver
   (merge (sql/IDriverSQLDefaultsMixin)
-         {:date-interval  date-interval
+         {:active-tables  sql/post-filtered-active-tables
+          :date-interval  date-interval
           :details-fields (constantly [{:name         "db"
                                         :display-name "Filename"
                                         :placeholder  "/home/camsaul/toucan_sightings.sqlite 😋"

src/metabase/driver/sync.clj

@@ -84,12 +84,12 @@
     (validate-active-tables active-tables)
 
     (mark-inactive-tables! database active-tables existing-table->id)
-    (create-new-tables!    database active-tables existing-table->id))
+    (create-new-tables!    database active-tables existing-table->id)
 
-  (fetch-and-sync-database-active-tables! driver database)
+    (fetch-and-sync-database-active-tables! driver database)
 
-  ;; Ok, now if we had a _metabase_metadata table from earlier we can go ahead and sync from it
-  (sync-metabase-metadata-table! driver database))
+    ;; Ok, now if we had a _metabase_metadata table from earlier we can go ahead and sync from it
+    (sync-metabase-metadata-table! driver database active-tables)))
 
 (defn- -sync-database-with-tracking! [driver database]
   (let [start-time    (System/currentTimeMillis)
@@ -125,8 +125,8 @@
    `keypath` is of the form `table-name.key` or `table-name.field-name.key`, where `key` is the name of some property of `Table` or `Field`.
 
    This functionality is currently only used by the Sample Dataset. In order to use this functionality, drivers must implement optional fn `:table-rows-seq`."
-  [driver database]
-  (doseq [{table-name :name} (driver/active-tables driver database)]
+  [driver database active-tables]
+  (doseq [{table-name :name} active-tables]
     (when (= (s/lower-case table-name) "_metabase_metadata")
       (doseq [{:keys [keypath value]} (driver/table-rows-seq driver database table-name)]
         (let [[_ table-name field-name k] (re-matches #"^([^.]+)\.(?:([^.]+)\.)?([^.]+)$" keypath)]