Skip to content
Snippets Groups Projects
Unverified Commit edf1c036 authored by Robert Roland's avatar Robert Roland Committed by GitHub
Browse files

Support for externally linked tables (#12741)

In Redshift (and other databases), foreign tables show up as "EXTERNAL
TABLE" - allow those to be synced in with Metabase.

Updates the Redshift JDBC driver due to changelog entry from Amazon that
was occurring and causing immense slowdowns in running the Redshift
driver tests:

    When the Java application uses com.amazon.redshift.api.PGTimestamp directly, an exception occurs.

Resolves #7833
parent ad33ebdb
No related branches found
No related tags found
Loading
......@@ -6,7 +6,7 @@
:dependencies
[[com.amazon.redshift/redshift-jdbc42-no-awssdk "1.2.43.1067"]]
[[com.amazon.redshift/redshift-jdbc42-no-awssdk "1.2.45.1069"]]
:profiles
{:provided
......
......@@ -60,3 +60,74 @@
:nested? false
:query-hash (byte-array [-53, -125, -44, -10, -18, -36, 37, 14, -37, 15, 44, 22, -8, -39, -94, 30, 93, 66, -13, 34, -52, -20, -31, 73, 76, -114, -13, -42, 52, 88, 31, -30])})))
"if I run a Redshift query, does it get a remark added to it?")))))
;; the extsales table is a Redshift Spectrum linked table, provided by AWS's sample data set for Redshift.
;; See https://docs.aws.amazon.com/redshift/latest/dg/c-getting-started-using-spectrum.html
;;
;; Data is loaded into S3 via:
;;
;; aws s3 cp s3://awssampledbuswest2/tickit/spectrum/sales/ s3://mb-rs-test/tickit/spectrum/sales/ --recursive
;;
;; And the Redshift table and schema is created via:
;;
;; create external schema spectrum
;; from data catalog
;; database 'spectrumdb'
;; iam_role ''
;; create external database if not exists;
;;
;; create external table spectrum.extsales(
;; salesid integer,
;; listid integer,
;; sellerid integer,
;; buyerid integer,
;; eventid integer,
;; dateid smallint,
;; qtysold smallint,
;; pricepaid decimal(8,2),
;; commission decimal(8,2),
;; saletime timestamp)
;; row format delimited
;; fields terminated by '\t'
;; stored as textfile
;; location 's3://mb-rs-test/tickit/spectrum/sales/'
;; table properties ('numRows'='172000');
;;
(deftest test-external-table
(mt/test-driver :redshift
(testing "expects spectrum schema to exist"
(is (= [{:description nil
:table_id (mt/id :extsales)
:special_type nil
:name "buyerid"
:settings nil
:source :fields
:field_ref [:field-id (mt/id :extsales :buyerid)]
:parent_id nil
:id (mt/id :extsales :buyerid)
:visibility_type :normal
:display_name "Buyer ID"
:base_type :type/Integer}
{:description nil
:table_id (mt/id :extsales)
:special_type nil
:name "salesid"
:settings nil
:source :fields
:field_ref [:field-id (mt/id :extsales :salesid)]
:parent_id nil
:id (mt/id :extsales :salesid)
:visibility_type :normal
:display_name "Sale Sid"
:base_type :type/Integer}]
; in different Redshift instances, the fingerprint on these
; columns is different.
(map #(dissoc % :fingerprint)
(get-in (qp/process-query (mt/mbql-query
:extsales
{:limit 1
:fields [$buyerid $salesid]
:order-by [[:asc $buyerid]
[:asc $salesid]]
:filter [:= [:field-id (mt/id :extsales :buyerid)] 11498]}))
[:data :cols])))))))
......@@ -101,7 +101,7 @@
[^DatabaseMetaData metadata ^String schema-or-nil ^String db-name-or-nil]
;; tablePattern "%" = match all tables
(with-open [rs (.getTables metadata db-name-or-nil schema-or-nil "%"
(into-array String ["TABLE" "VIEW" "FOREIGN TABLE" "MATERIALIZED VIEW"]))]
(into-array String ["TABLE" "VIEW" "FOREIGN TABLE" "MATERIALIZED VIEW" "EXTERNAL TABLE"]))]
(mapv #(select-keys % [:table_name :remarks :table_schem])
(jdbc/result-set-seq rs))))
......
......@@ -95,7 +95,7 @@
;;; | Loading Test Extensions |
;;; +----------------------------------------------------------------------------------------------------------------+
(declare before-run after-run)
(declare before-run)
(defonce ^:private has-done-before-run (atom #{}))
......@@ -247,7 +247,6 @@
(defmethod before-run ::test-extensions [_]) ; default-impl is a no-op
(defmulti dbdef->connection-details
"Return the connection details map that should be used to connect to the Database we will create for
`database-definition`.
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment