reorganize sync.clj a bit

src/metabase/driver/generic_sql/sync.clj

@@ -8,6 +8,52 @@
             (metabase.models [field :refer [Field]]
                              [table :refer [Table]])))
 
+(declare sync-fields
+         table-names
+         update-table-row-count)
+
+;; # PUBLIC FUNCTIONS
+
+(defn sync-database
+  [{:keys [id] :as database}]
+  (with-jdbc-metadata database                                                                ; with-jdbc-metadata reuses *jdbc-metadata* in any call to it inside the fn passed to it
+    (fn [_]                                                                                    ; by wrapping the entire sync operation in this we can reuse the same connection throughout
+      (->> (table-names database)
+        (pmap (fn [table-name]
+                (binding [*entity-overrides* {:transforms [#(assoc % :db (delay database))]}] ; add a korma transform to Table that will assoc :db on results.
+                  (let [table (or (sel :one Table :db_id id :name table-name)                 ; Table's post-select only sets :db if it's not already set.
+                                  (ins Table                                                  ; This way, we can reuse a single `database` instead of creating
+                                    :db_id id                                                 ; a few dozen duplicate instances of it.
+                                    :name table-name                                          ; We can re-use one korma connection pool instead of
+                                    :active true))]                                           ; creating dozens of them, which was causing issues with too
+                    (update-table-row-count table)                                            ; many open connections.
+                    (sync-fields table)
+                    (log/debug "Synced" table-name)))))
+        dorun))))
+
+(defn sync-table
+  [{:keys [db] :as table}]
+  (with-jdbc-metadata @db
+    (fn [_]
+      (update-table-row-count table)
+      (sync-fields table)
+      (log/debug "Synced" (:name table)))))
+
+
+;; # IMPLEMENTATION
+
+;; ## Variables
+
+(def ^:const ^:private low-cardinality-threshold
+  "Fields with less than this many distinct values should automatically be marked with `special_type` `:category`."
+  40)
+
+(def ^:dynamic *column->base-type*
+  "COLUMN->BASE-TYPE should be a map of column types returned by the DB to Field base types."
+  {})
+
+
+;; ## Fetch Tables/Columns
 
 (defn table-names
   "Fetch a list of table names for DATABASE."
@@ -25,8 +71,9 @@
     (fn [md] (->> (-> md
                     (.getColumns nil nil table-name nil) ; ResultSet getColumns(String catalog, String schemaPattern, String tableNamePattern, String columnNamePattern)
                     jdbc/result-set-seq)
-               (mapv #(select-keys % [:column_name :type_name]))))))
+                 (mapv #(select-keys % [:column_name :type_name]))))))
 
+;; ## Table Info
 
 (defn get-table-row-count
   "Get the number of rows in TABLE."
@@ -43,9 +90,8 @@
   (let [new-count (get-table-row-count table)]
     (upd Table id :rows new-count)))
 
-(def ^:dynamic *column->base-type*
-  "COLUMN->BASE-TYPE should be a map of column types returned by the DB to Field base types."
-  {})
+
+;; ## Field Info
 
 (defn sync-fields
   "Sync `Fields` for TABLE. "
@@ -59,28 +105,3 @@
                                      (throw (Exception. (str "Column '" column_name "' has an unknown type: '" type_name
                                                              "'. Please add the type mapping to corresponding driver (e.g. metabase.driver.postgres.sync).")))))))
               (jdbc-columns db name))))
-
-(defn sync-database
-  [{:keys [id] :as database}]
-  (with-jdbc-metadata database                                                                ; with-jdbc-metadata reuses *jdbc-metadata* in any call to it inside the fn passed to it
-    (fn [_]                                                                                    ; by wrapping the entire sync operation in this we can reuse the same connection throughout
-      (->> (table-names database)
-        (pmap (fn [table-name]
-                (binding [*entity-overrides* {:transforms [#(assoc % :db (delay database))]}] ; add a korma transform to Table that will assoc :db on results.
-                  (let [table (or (sel :one Table :db_id id :name table-name)                 ; Table's post-select only sets :db if it's not already set.
-                                  (ins Table                                                  ; This way, we can reuse a single `database` instead of creating
-                                    :db_id id                                                 ; a few dozen duplicate instances of it.
-                                    :name table-name                                          ; We can re-use one korma connection pool instead of
-                                    :active true))]                                           ; creating dozens of them, which was causing issues with too
-                    (update-table-row-count table)                                            ; many open connections.
-                    (sync-fields table)
-                    (log/debug "Synced" table-name)))))
-        dorun))))
-
-(defn sync-table
-  [{:keys [db] :as table}]
-  (with-jdbc-metadata @db
-    (fn [_]
-      (update-table-row-count table)
-      (sync-fields table)
-      (log/debug "Synced" (:name table)))))