diff --git a/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk_test.clj b/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk_test.clj index b82117afd730c9e909cdea4d0d467bea3ee47599..2001bd1e9153b63e9facc93ea105d6fbf354188e 100644 --- a/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk_test.clj +++ b/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk_test.clj @@ -616,13 +616,10 @@ (is (= max-rows (count rows))) (is (= (/ max-rows page-size) @num-page-callbacks))))))))) -(defn- sync-and-assert-filtered-tables [database assert-table-fn] - (t2.with-temp/with-temp [Database db-filtered database] - (sync/sync-database! db-filtered {:scan :schema}) - (let [tables (t2/select Table :db_id (u/the-id db-filtered))] - (assert (not-empty tables)) - (doseq [table tables] - (assert-table-fn table))))) +(defn- synced-tables [db-attributes] + (t2.with-temp/with-temp [Database db db-attributes] + (sync/sync-database! db {:scan :schema}) + (t2/select Table :db_id (u/the-id db)))) (deftest dataset-filtering-test (mt/test-driver :bigquery-cloud-sdk @@ -637,33 +634,42 @@ (dissoc :dataset-filters-type :dataset-filters-patterns))) dataset-ids (map #(.getDataset %) datasets) - ;; get the first 4 characters of each dataset-id + ;; get the first 4 characters of each dataset-id. The first 4 characters are used because the first 3 are + ;; often used for bigquery dataset names e.g. `v4_test_data` prefixes (->> dataset-ids (map (fn [dataset-id] (apply str (take 4 dataset-id))))) - ;; inclusion-patterns selects the first dataset - ;; exclusion-patterns excludes almost everything else include-prefix (first prefixes) + exclude-prefixes (rest prefixes) + ;; inclusion-patterns selects the first dataset inclusion-patterns (str include-prefix "*") - exclusion-patterns (str/join "," (map #(str % "*") (set (rest prefixes))))] + ;; exclusion-patterns excludes every dataset with exclude prefixes. It would exclude all other datasets + ;; except ones that match the include-prefix, except there could be other tests creating new datasets for + ;; the same test DB while this test is running, so we can't guarantee that the include-prefix will match all the + ;; datasets + exclusion-patterns (str/join "," (map #(str % "*") (set exclude-prefixes)))] (testing " with an inclusion filter" - (sync-and-assert-filtered-tables {:name "BigQuery Test DB with dataset inclusion filters" - :engine :bigquery-cloud-sdk - :details (-> (mt/db) - :details - (assoc :dataset-filters-type "inclusion" - :dataset-filters-patterns inclusion-patterns))} - (fn [{dataset-id :schema}] - (is (str/starts-with? dataset-id include-prefix))))) + (let [tables (synced-tables {:name "BigQuery Test DB with dataset inclusion filters" + :engine :bigquery-cloud-sdk + :details (-> (mt/db) + :details + (assoc :dataset-filters-type "inclusion" + :dataset-filters-patterns inclusion-patterns))})] + (is (seq tables)) + (doseq [{dataset-id :schema} tables] + (is (str/starts-with? dataset-id include-prefix))))) (testing " with an exclusion filter" - (sync-and-assert-filtered-tables {:name "BigQuery Test DB with dataset exclusion filters" - :engine :bigquery-cloud-sdk - :details (-> (mt/db) - :details - (assoc :dataset-filters-type "exclusion" - :dataset-filters-patterns exclusion-patterns))} - (fn [{dataset-id :schema}] - (is (str/starts-with? dataset-id include-prefix)))))))))) + (let [tables (synced-tables {:name "BigQuery Test DB with dataset inclusion filters" + :engine :bigquery-cloud-sdk + :details (-> (mt/db) + :details + (assoc :dataset-filters-type "exclusion" + :dataset-filters-patterns exclusion-patterns))})] + (testing "\ncheck that all synced tables do not start with any of the exclude prefixes" + (doseq [{dataset-id :schema} tables] + (is (not (some #(str/starts-with? dataset-id %) exclude-prefixes))))) + (testing "\ncheck that the dataset with the include prefix is not excluded" + (is (some #(str/starts-with? (:schema %) include-prefix) tables)))))))))) (deftest normalize-away-dataset-id-test (mt/test-driver :bigquery-cloud-sdk