[MLv2] Add matrix tests for filter functions (#37759)

e2e/test/scenarios/admin/settings/email.cy.spec.js

@@ -109,7 +109,9 @@ describe("scenarios > admin > settings > email settings", () => {
     });
     cy.visit("/admin/settings/email/smtp");
     main().findByText("Send test email").click();
-    cy.findAllByText("Couldn't connect to host, port: localhost, 1234; timeout -1");
+    cy.findAllByText(
+      "Couldn't connect to host, port: localhost, 1234; timeout -1",
+    );
   });
 
   it(

test/metabase/lib/filter_test.cljc

@@ -7,8 +7,10 @@
    [metabase.lib.core :as lib]
    [metabase.lib.filter.operator :as lib.filter.operator]
    [metabase.lib.metadata :as lib.metadata]
+   [metabase.lib.options :as lib.options]
    [metabase.lib.test-metadata :as meta]
    [metabase.lib.test-util :as lib.tu]
+   [metabase.lib.test-util.matrix :as matrix]
    [metabase.lib.types.isa :as lib.types.isa]
    [metabase.util :as u]))
 
@@ -730,3 +732,24 @@
                  (map (comp signature-fn
                             #(lib/display-info filtered-query %))
                       filtered-query-cols))))))))
+
+(deftest ^:parallel matrix-can-filter
+  (doseq [{:keys [column-type v]} #{{:column-type :type/DateTimeWithLocalTZ :v "2014-01-01"}
+                                    {:column-type :type/Integer :v 1}
+                                    {:column-type :type/Text :v "str"}
+                                    {:column-type :type/Boolean :v true}
+                                    {:column-type :type/Coordinate :v 1}}
+          [query desired] (matrix/test-queries column-type)]
+    (let [col (matrix/find-first desired (lib/filterable-columns query))
+          query' (lib/filter query (lib/= col v))
+          parts (lib/expression-parts query' (first (lib/filters query')))]
+      (is (=? (lib.filter.operator/filter-operators {:lib/type :metadata/column :name "expected" :base-type column-type})
+              (lib/filterable-column-operators col)))
+      (is (=? [[:= {} (lib.options/update-options (lib/ref col) dissoc :lib/uuid) v]]
+              (lib/filters query')))
+      (is (=? {:operator :=
+               :args [{:lib/type :metadata/column
+                       :name (:name col)} v]}
+              parts))
+      (is (=? [:= {} (lib.options/update-options (lib/ref col) dissoc :lib/uuid) v]
+              (lib/expression-clause (:operator parts) (:args parts) nil))))))

test/metabase/lib/test_util.cljc

@@ -190,38 +190,42 @@
                                                     :semantic-type :type/FK}]}
                    :native             "SELECT whatever"}]})
 
+(defn make-mock-cards
+  "Create mock cards against a set of tables in meta/metadata-provider. See [[mock-cards]]"
+  [metadata-provider table-key-and-ids]
+  (into {}
+        (comp (mapcat (fn [[table table-id]]
+                        [{:table table, :table-id table-id :metadata? true,  :native? false, :card-name table}
+                         {:table table, :table-id table-id :metadata? true,  :native? true,  :card-name (keyword (name table) "native")}
+                         {:table table, :table-id table-id :metadata? false, :native? false, :card-name (keyword (name table) "no-metadata")}]))
+              (map-indexed (fn [idx {:keys [table table-id metadata? native? card-name]}]
+                             [card-name
+                              (merge
+                                {:lib/type      :metadata/card
+                                 :id            (inc idx)
+                                 :database-id   (:id (lib.metadata/database metadata-provider))
+                                 :name          (str "Mock " (name table) " card")
+                                 :dataset-query (if native?
+                                                  {:database (:id (lib.metadata/database metadata-provider))
+                                                   :type     :native
+                                                   :native   {:query (str "SELECT * FROM " (name table))}}
+                                                  {:database (:id (lib.metadata/database metadata-provider))
+                                                   :type     :query
+                                                   :query    {:source-table table-id}})}
+                                (when metadata?
+                                  {:result-metadata
+                                   (->> (lib.metadata/fields metadata-provider table-id)
+                                        (sort-by :id)
+                                        (mapv #(if native? (dissoc % :table-id :id :fk-target-field-id) %)))}))])))
+        table-key-and-ids))
+
 (def mock-cards
   "Map of mock MBQL query Card against the test tables. There are three versions of the Card for each table:
 
   * `:venues`, a Card WITH `:result-metadata`
   * `:venues/no-metadata`, a Card WITHOUT `:result-metadata`
   * `:venues/native`, a Card with `:result-metadata` and a NATIVE query."
-  (into {}
-        (comp (mapcat (fn [table]
-                        [{:table table, :metadata? true,  :native? false, :card-name table}
-                         {:table table, :metadata? true,  :native? true,  :card-name (keyword (name table) "native")}
-                         {:table table, :metadata? false, :native? false, :card-name (keyword (name table) "no-metadata")}]))
-              (map-indexed (fn [idx {:keys [table metadata? native? card-name]}]
-                             [card-name
-                              (merge
-                               {:lib/type      :metadata/card
-                                :id            (inc idx)
-                                :database-id   (meta/id)
-                                :name          (str "Mock " (name table) " card")
-                                :dataset-query (if native?
-                                                 {:database (meta/id)
-                                                  :type     :native
-                                                  :native   {:query (str "SELECT * FROM " (name table))}}
-                                                 {:database (meta/id)
-                                                  :type     :query
-                                                  :query    {:source-table (meta/id table)}})}
-                               (when metadata?
-                                 {:result-metadata
-                                  (->> (meta/fields table)
-                                       (map (partial meta/field-metadata table))
-                                       (sort-by :id)
-                                       (mapv #(if native? (dissoc % :table-id :id :fk-target-field-id) %)))}))])))
-        (meta/tables)))
+  (make-mock-cards meta/metadata-provider (map (juxt identity (comp :id meta/table-metadata)) (meta/tables))))
 
 (defn metadata-provider-with-mock-card [card]
   (lib/composed-metadata-provider

test/metabase/lib/test_util/matrix.cljc

+(ns metabase.lib.test-util.matrix
+  (:require
+    [clojure.test :refer [deftest is]]
+    [medley.core :as m]
+    [metabase.lib.core :as lib]
+    [metabase.lib.metadata :as lib.metadata]
+    [metabase.lib.test-metadata :as meta]
+    [metabase.lib.test-metadata.graph-provider :as meta.graph-provider]
+    [metabase.lib.test-util :as lib.tu]
+    [metabase.lib.test-util.metadata-providers.mock :as providers.mock]))
+
+(defn- metadata-with-column-of-type
+  [column-type]
+  (let [test-column {:description nil
+                     :lib/type :metadata/column
+                     :database-is-auto-increment false
+                     :fingerprint-version 5
+                     :base-type column-type
+                     :database-required false
+                     :table-id 1
+                     :name "TEST_ME"
+                     :coercion-strategy nil
+                     :settings nil
+                     :caveats nil
+                     :nfc-path nil
+                     ;;:database-type "TIMESTAMP WITH TIME ZONE"
+                     :effective-type column-type
+                     :fk-target-field-id nil
+                     :custom-position 0
+                     :active true
+                     :id 2
+                     :parent-id nil
+                     :points-of-interest nil
+                     :visibility-type :normal
+                     :display-name "Test Me"
+                     :position 1
+                     :has-field-values nil
+                     :preview-display true
+                     :database-position 1}]
+   (assoc (assoc (dissoc meta/metadata :tables) :id 1)
+         :tables
+         [{:description nil,
+           :schema "PUBLIC",
+           :lib/type :metadata/table,
+           :entity-type :entity/GenericTable,
+           :db-id 1,
+           :name "MATRIX",
+           :initial-sync-status "complete",
+           :caveats nil,
+           :field-order :database,
+           :show-in-getting-started false,
+           :active true,
+           :id 1 ,
+           :points-of-interest nil,
+           :visibility-type nil,
+           :display-name "Matrix"
+           :fields [{:description nil,
+                     :lib/type :metadata/column,
+                     :fingerprint-version 0,
+                     :base-type :type/BigInteger,
+                     :semantic-type :type/PK,
+                     :database-required false,
+                     :table-id 1,
+                     :name "ID",
+                     :coercion-strategy nil,
+                     :settings nil,
+                     :caveats nil,
+                     :nfc-path nil,
+                     :database-type "BIGINT",
+                     :effective-type :type/BigInteger,
+                     :fk-target-field-id nil,
+                     :custom-position 0,
+                     :active true,
+                     :id 1,
+                     :parent-id nil,
+                     :points-of-interest nil,
+                     :visibility-type :normal,
+                     :display-name "ID",
+                     :position 0,
+                     :has-field-values :none,
+                     :target nil,
+                     :preview-display true,
+                     :database-position 0,
+                     :fingerprint nil}
+                    test-column]}
+          {:description nil,
+           :schema "PUBLIC",
+           :lib/type :metadata/table,
+           :entity-type :entity/GenericTable,
+           :db-id 1,
+           :name "SUPPORT",
+           :initial-sync-status "complete",
+           :caveats nil,
+           :field-order :database,
+           :show-in-getting-started false,
+           :active true,
+           :id 10,
+           :points-of-interest nil,
+           :visibility-type nil,
+           :display-name "Support"
+           :fields [{:description nil,
+                     :lib/type :metadata/column,
+                     :fingerprint-version 0,
+                     :base-type :type/BigInteger,
+                     :semantic-type :type/PK,
+                     :database-required false,
+                     :table-id 10,
+                     :name "ID",
+                     :coercion-strategy nil,
+                     :settings nil,
+                     :caveats nil,
+                     :nfc-path nil,
+                     :database-type "BIGINT",
+                     :effective-type :type/BigInteger,
+                     :fk-target-field-id nil,
+                     :custom-position 0,
+                     :active true,
+                     :id 10,
+                     :parent-id nil,
+                     :points-of-interest nil,
+                     :visibility-type :normal,
+                     :display-name "ID",
+                     :position 0,
+                     :has-field-values :none,
+                     :target nil,
+                     :preview-display true,
+                     :database-position 0,
+                     :fingerprint nil}
+                    {:description nil
+                     :lib/type :metadata/column
+                     :database-is-auto-increment false
+                     :fingerprint-version 5
+                     :base-type :type/Integer
+                     :semantic-type :type/FK
+                     :database-required false
+                     :table-id 10
+                     :name "FK_ID"
+                     :coercion-strategy nil
+                     :settings nil
+                     :caveats nil
+                     :nfc-path nil
+                     :database-type "INTEGER"
+                     :effective-type :type/Integer
+                     :fk-target-field-id 1
+                     :custom-position 0
+                     :active true
+                     :id 20
+                     :parent-id nil
+                     :points-of-interest nil
+                     :visibility-type :normal
+                     :display-name "Foreign ID"
+                     :position 1
+                     :has-field-values nil
+                     :preview-display true
+                     :database-position 1
+                     :fingerprint {:global {:distinct-count 200
+                                            :nil% 0.0}}}]}])))
+
+(defn test-queries
+  "Produces test queries that will contain a column of the specified type.
+   The queries are various ways that a database column could be manipulated to expose such a returned-column type.
+   Returns a `[query desired-column-alias]` tuple such that `desired-column-alias` is the name of the visible-column that you can use along with `matrix/find-first` to find your column under test."
+  [column-type]
+  (let [mp (meta.graph-provider/->SimpleGraphMetadataProvider (metadata-with-column-of-type column-type))
+        mock-cards (lib.tu/make-mock-cards mp [[:matrix 1]])
+        mp (lib/composed-metadata-provider
+             mp
+             (providers.mock/mock-metadata-provider
+               {:cards (vals mock-cards)}))
+        table-meta (lib.metadata/table mp 1)
+        field-meta (lib.metadata/field mp 2)
+        queries [[(lib/query mp table-meta)
+                  "TEST_ME"]
+                 [(-> (lib/query mp table-meta)
+                      (lib/append-stage)
+                      (lib/append-stage))
+                  "TEST_ME"]
+                 [(-> (lib/query mp table-meta)
+                      (lib/expression "expr" field-meta))
+                  "expr"]
+                 [(-> mp
+                      (lib/query table-meta)
+                      (lib/breakout field-meta))
+                  "TEST_ME"]
+                 [(-> mp
+                      (lib/query table-meta)
+                      (lib/breakout field-meta)
+                      (lib/append-stage))
+                  "TEST_ME"]
+                 [(lib/query mp (lib.metadata/table mp 10))
+                  "MATRIX__via__FK_ID__TEST_ME"]
+                 [(-> (lib/query mp (lib.metadata/table mp 10))
+                      (lib/join table-meta))
+                  "Matrix - Foreign__TEST_ME"]
+                 [(-> (lib/query mp table-meta)
+                      (lib/join (lib/join-clause table-meta [(lib/= (lib.metadata/field mp 1) (lib.metadata/field mp 1))])))
+                  "Matrix__TEST_ME"]
+                 [(lib/query mp (:matrix mock-cards))
+                  "TEST_ME"]
+                 [(lib/query mp (:matrix/native mock-cards))
+                  "TEST_ME"]]]
+    queries))
+
+(defn find-first
+  "Finds the column with the matching `:lib/desired-column-alias`"
+  [desired columns]
+  (m/find-first (comp #(= desired %) :lib/desired-column-alias) columns))
+
+(deftest ^:parallel matrix-test-queries-test
+  (doseq [[query desired] (test-queries :type/Text)]
+    (is (find-first desired (lib/visible-columns query)))))