Skip to content
Snippets Groups Projects
Commit eaf36711 authored by Allen Gilliland's avatar Allen Gilliland
Browse files

Merge pull request #2422 from metabase/bigquery-paging 

Handle paging when fetching BigQuery metadata :scream_cat:
parents a4bd56fd 0aed082f
Branches
Tags
No related merge requests found
Hide whitespace changes
Inline Side-by-side
src/metabase/driver/bigquery.clj
+ 20
− 8
View file @ eaf36711
......@@ -86,20 +86,32 @@
(def ^:private ^{:arglists '([database])} ^Bigquery database->client (comp credential->client database->credential))
(defn- ^TableList list-tables
([{{:keys [project-id dataset-id]} :details, :as database}]
(list-tables (database->client database) project-id dataset-id))
"Fetch a page of Tables. By default, fetches the first page; page size is 50. For cases when more than 50 Tables are present, you may
fetch subsequent pages by specifying the PAGE-TOKEN; the token for the next page is returned with a page when one exists."
([database]
(list-tables database nil))
([{{:keys [project-id dataset-id]} :details, :as database}, ^String page-token-or-nil]
(list-tables (database->client database) project-id dataset-id page-token-or-nil))
([^Bigquery client, ^String project-id, ^String dataset-id]
([^Bigquery client, ^String project-id, ^String dataset-id, ^String page-token-or-nil]
{:pre [client (seq project-id) (seq dataset-id)]}
(execute (.list (.tables client) project-id dataset-id))))
(execute (u/prog1 (.list (.tables client) project-id dataset-id)
(.setPageToken <> page-token-or-nil)))))
(defn- describe-database [database]
{:pre [(map? database)]}
{:tables (set (for [^TableList$Tables table (.getTables (list-tables database))
:let [^TableReference tableref (.getTableReference table)]]
{:schema nil, :name (.getTableId tableref)}))})
;; first page through all the 50-table pages until we stop getting "next page tokens"
(let [tables (loop [tables [], ^TableList table-list (list-tables database)]
(let [tables (concat tables (.getTables table-list))]
(if-let [next-page-token (.getNextPageToken table-list)]
(recur tables (list-tables database next-page-token))
tables)))]
;; after that convert the results to MB format
{:tables (set (for [^TableList$Tables table tables
:let [^TableReference tableref (.getTableReference table)]]
{:schema nil, :name (.getTableId tableref)}))}))
(defn- can-connect? [details-map]
{:pre [(map? details-map)]}
......
src/metabase/sync_database/analyze.clj
+ 5
− 5
View file @ eaf36711
(ns metabase.sync-database.analyze
"Functions which handle the in-depth data shape analysis portion of the sync process."
(:require [cheshire.core :as json]
[clojure.math.numeric-tower :as math]
(:require [clojure.math.numeric-tower :as math]
[clojure.string :as s]
[clojure.tools.logging :as log]
[cheshire.core :as json]
[korma.core :as k]
[schema.core :as schema]
[metabase.db :as db]
[metabase.db.metadata-queries :as queries]
[metabase.driver :as driver]
[metabase.models.field :as field]
[metabase.models.field-values :as field-values]
[metabase.models.table :as table]
(metabase.models [field :as field]
[field-values :as field-values]
[table :as table])
[metabase.sync-database.interface :as i]
[metabase.util :as u]))
......
src/metabase/sync_database/introspect.clj
+ 6
− 6
View file @ eaf36711
......@@ -2,13 +2,13 @@
"Functions which handle the raw sync process."
(:require [clojure.set :as set]
[clojure.tools.logging :as log]
[korma.core :as k]
[korma.db :as kdb]
(korma [core :as k]
[db :as kdb])
[schema.core :as schema]
[metabase.db :as db]
[metabase.driver :as driver]
[metabase.models.raw-column :as raw-column]
[metabase.models.raw-table :as raw-table]
(metabase [db :as db]
[driver :as driver])
(metabase.models [raw-column :as raw-column]
[raw-table :as raw-table])
[metabase.sync-database.interface :as i]
[metabase.util :as u]))
......
test/metabase/driver/bigquery_test.clj 0 → 100644
+ 66
− 0
View file @ eaf36711
(ns metabase.driver.bigquery-test
(:require metabase.driver.bigquery
[metabase.models.database :as database]
[metabase.test.data :as data]
(metabase.test.data [datasets :refer [expect-with-engine]]
[interface :refer [def-database-definition]])))
;; Make sure that paging works correctly for the bigquery driver when fetching a list of tables
;; Default page size is 50 so if we have more than that number of tables make sure they all show up
(def-database-definition ^:private fifty-one-different-tables
["birds_1" [{:field-name "name", :base-type :TextField}] [["Rasta"] ["Lucky"]]]
["birds_2" [{:field-name "name", :base-type :TextField}] [["Rasta"] ["Lucky"]]]
["birds_3" [{:field-name "name", :base-type :TextField}] [["Rasta"] ["Lucky"]]]
["birds_4" [{:field-name "name", :base-type :TextField}] [["Rasta"] ["Lucky"]]]
["birds_5" [{:field-name "name", :base-type :TextField}] [["Rasta"] ["Lucky"]]]
["birds_6" [{:field-name "name", :base-type :TextField}] [["Rasta"] ["Lucky"]]]
["birds_7" [{:field-name "name", :base-type :TextField}] [["Rasta"] ["Lucky"]]]
["birds_8" [{:field-name "name", :base-type :TextField}] [["Rasta"] ["Lucky"]]]
["birds_9" [{:field-name "name", :base-type :TextField}] [["Rasta"] ["Lucky"]]]
["birds_10" [{:field-name "name", :base-type :TextField}] [["Rasta"] ["Lucky"]]]
["birds_11" [{:field-name "name", :base-type :TextField}] [["Rasta"] ["Lucky"]]]
["birds_12" [{:field-name "name", :base-type :TextField}] [["Rasta"] ["Lucky"]]]
["birds_13" [{:field-name "name", :base-type :TextField}] [["Rasta"] ["Lucky"]]]
["birds_14" [{:field-name "name", :base-type :TextField}] [["Rasta"] ["Lucky"]]]
["birds_15" [{:field-name "name", :base-type :TextField}] [["Rasta"] ["Lucky"]]]
["birds_16" [{:field-name "name", :base-type :TextField}] [["Rasta"] ["Lucky"]]]
["birds_17" [{:field-name "name", :base-type :TextField}] [["Rasta"] ["Lucky"]]]
["birds_18" [{:field-name "name", :base-type :TextField}] [["Rasta"] ["Lucky"]]]
["birds_19" [{:field-name "name", :base-type :TextField}] [["Rasta"] ["Lucky"]]]
["birds_20" [{:field-name "name", :base-type :TextField}] [["Rasta"] ["Lucky"]]]
["birds_21" [{:field-name "name", :base-type :TextField}] [["Rasta"] ["Lucky"]]]
["birds_22" [{:field-name "name", :base-type :TextField}] [["Rasta"] ["Lucky"]]]
["birds_23" [{:field-name "name", :base-type :TextField}] [["Rasta"] ["Lucky"]]]
["birds_24" [{:field-name "name", :base-type :TextField}] [["Rasta"] ["Lucky"]]]
["birds_25" [{:field-name "name", :base-type :TextField}] [["Rasta"] ["Lucky"]]]
["birds_26" [{:field-name "name", :base-type :TextField}] [["Rasta"] ["Lucky"]]]
["birds_27" [{:field-name "name", :base-type :TextField}] [["Rasta"] ["Lucky"]]]
["birds_28" [{:field-name "name", :base-type :TextField}] [["Rasta"] ["Lucky"]]]
["birds_29" [{:field-name "name", :base-type :TextField}] [["Rasta"] ["Lucky"]]]
["birds_30" [{:field-name "name", :base-type :TextField}] [["Rasta"] ["Lucky"]]]
["birds_31" [{:field-name "name", :base-type :TextField}] [["Rasta"] ["Lucky"]]]
["birds_32" [{:field-name "name", :base-type :TextField}] [["Rasta"] ["Lucky"]]]
["birds_33" [{:field-name "name", :base-type :TextField}] [["Rasta"] ["Lucky"]]]
["birds_34" [{:field-name "name", :base-type :TextField}] [["Rasta"] ["Lucky"]]]
["birds_35" [{:field-name "name", :base-type :TextField}] [["Rasta"] ["Lucky"]]]
["birds_36" [{:field-name "name", :base-type :TextField}] [["Rasta"] ["Lucky"]]]
["birds_37" [{:field-name "name", :base-type :TextField}] [["Rasta"] ["Lucky"]]]
["birds_38" [{:field-name "name", :base-type :TextField}] [["Rasta"] ["Lucky"]]]
["birds_39" [{:field-name "name", :base-type :TextField}] [["Rasta"] ["Lucky"]]]
["birds_40" [{:field-name "name", :base-type :TextField}] [["Rasta"] ["Lucky"]]]
["birds_41" [{:field-name "name", :base-type :TextField}] [["Rasta"] ["Lucky"]]]
["birds_42" [{:field-name "name", :base-type :TextField}] [["Rasta"] ["Lucky"]]]
["birds_43" [{:field-name "name", :base-type :TextField}] [["Rasta"] ["Lucky"]]]
["birds_44" [{:field-name "name", :base-type :TextField}] [["Rasta"] ["Lucky"]]]
["birds_45" [{:field-name "name", :base-type :TextField}] [["Rasta"] ["Lucky"]]]
["birds_46" [{:field-name "name", :base-type :TextField}] [["Rasta"] ["Lucky"]]]
["birds_47" [{:field-name "name", :base-type :TextField}] [["Rasta"] ["Lucky"]]]
["birds_48" [{:field-name "name", :base-type :TextField}] [["Rasta"] ["Lucky"]]]
["birds_49" [{:field-name "name", :base-type :TextField}] [["Rasta"] ["Lucky"]]]
["birds_50" [{:field-name "name", :base-type :TextField}] [["Rasta"] ["Lucky"]]]
["birds_51" [{:field-name "name", :base-type :TextField}] [["Rasta"] ["Lucky"]]])
(expect-with-engine :bigquery
51
(data/with-temp-db [db fifty-one-different-tables]
(count (database/tables db))))
test/metabase/test/data.clj
+ 6
− 6
View file @ eaf36711
......@@ -244,12 +244,12 @@
(let [loader *data-loader*
dbdef (i/map->DatabaseDefinition (assoc dbdef :short-lived? true))]
(swap! loader->loaded-db-def conj [loader dbdef])
(with-db (binding [*sel-disable-logging* true]
(let [db (get-or-create-database! loader dbdef)]
(assert db)
(assert (exists? Database :id (:id db)))
db))
(f db))))
(binding [*sel-disable-logging* true]
(let [db (get-or-create-database! loader dbdef)]
(assert db)
(assert (exists? Database :id (:id db)))
(with-db db
(f db))))))
(defmacro with-temp-db
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment