Spark SQL driver: read Hive database schema and partitionned tables (#9889)

modules/drivers/sparksql/src/metabase/driver/sparksql.clj

@@ -93,11 +93,17 @@
   {:tables
    (with-open [conn (jdbc/get-connection (sql-jdbc.conn/db->pooled-connection-spec database))]
      (set
-      (for [{:keys [tablename database]} (jdbc/query {:connection conn} ["show tables"])]
-        {:name   tablename
+      (for [{:keys [database tablename tab_name]} (jdbc/query {:connection conn} ["show tables"])]
+        {:name   (or tablename tab_name) ; column name differs depending on server (SparkSQL, hive, Impala)
          :schema (when (seq database)
                    database)})))})
 
+;; Hive describe table result has commented rows to distinguish partitions
+(defn- valid-describe-table-row? [{:keys [col_name data_type]}]
+  (every? (every-pred (complement str/blank?)
+                      (complement #(str/starts-with? % "#")))
+          [col_name data_type]))
+
 ;; workaround for SPARK-9686 Spark Thrift server doesn't return correct JDBC metadata
 (defmethod driver/describe-table :sparksql
   [driver {:keys [details] :as database} {table-name :name, schema :schema, :as table}]
@@ -111,7 +117,8 @@
                                                       (dash-to-underscore schema)
                                                       (dash-to-underscore table-name)))])]
        (set
-        (for [{col-name :col_name, data-type :data_type} results]
+        (for [{col-name :col_name, data-type :data_type, :as result} results
+              :when                                                  (valid-describe-table-row? result)]
           {:name          col-name
            :database-type data-type
            :base-type     (sql-jdbc.sync/database-type->base-type :hive-like (keyword data-type))}))))})