diff --git a/frontend/src/metabase-types/api/search.ts b/frontend/src/metabase-types/api/search.ts index f34d27a7890d1a8fdecf20303b863565ea2c4da1..5623007b2a7d08f6b3f9e7d750d8b38423d88ab3 100644 --- a/frontend/src/metabase-types/api/search.ts +++ b/frontend/src/metabase-types/api/search.ts @@ -42,7 +42,6 @@ export interface SearchScore { | "text-total-occurrences" | "text-fullness"; match?: string; - "match-context-thunk"?: string; column?: string; } diff --git a/frontend/src/metabase/static-viz/components/ComboChart/stories-data/line-unpin-from-zero.json b/frontend/src/metabase/static-viz/components/ComboChart/stories-data/line-unpin-from-zero.json index 1edf0c4f352609889dbdf4a285a71deba0048573..f7e0f103eba0e3a558d0373cc581f1b41818b990 100644 --- a/frontend/src/metabase/static-viz/components/ComboChart/stories-data/line-unpin-from-zero.json +++ b/frontend/src/metabase/static-viz/components/ComboChart/stories-data/line-unpin-from-zero.json @@ -37,7 +37,6 @@ "name": "text-consecutivity", "weight": 4, "match": "line-unpin-from-zero", - "match-context-thunk": "metabase.search.scoring$text_scores_with$iter__130228__130234$fn__130235$iter__130230__130236$fn__130237$fn__130238$fn__130241@720ce99c", "column": "name" }, { @@ -45,7 +44,6 @@ "name": "text-total-occurrences", "weight": 4, "match": "line-unpin-from-zero", - "match-context-thunk": "metabase.search.scoring$text_scores_with$iter__130228__130234$fn__130235$iter__130230__130236$fn__130237$fn__130238$fn__130241@6e4a4973", "column": "name" }, { @@ -53,7 +51,6 @@ "name": "text-fullness", "weight": 2, "match": "line-unpin-from-zero", - "match-context-thunk": "metabase.search.scoring$text_scores_with$iter__130228__130234$fn__130235$iter__130230__130236$fn__130237$fn__130238$fn__130241@f75a6de", "column": "name" } ], diff --git a/frontend/src/metabase/static-viz/components/ComboChart/stories-data/ticks-native-week-with-gap-long-range.json b/frontend/src/metabase/static-viz/components/ComboChart/stories-data/ticks-native-week-with-gap-long-range.json index 4cb90e24975fe18082beb2d88cad7c7507f9cf0a..10f34a0bca7cf71982490756bed235351d66a971 100644 --- a/frontend/src/metabase/static-viz/components/ComboChart/stories-data/ticks-native-week-with-gap-long-range.json +++ b/frontend/src/metabase/static-viz/components/ComboChart/stories-data/ticks-native-week-with-gap-long-range.json @@ -61,7 +61,6 @@ "name": "text-consecutivity", "weight": 4, "match": "x-axis: native weekly", - "match-context-thunk": "metabase.search.scoring$text_scores_with$iter__126238__126244$fn__126245$iter__126240__126246$fn__126247$fn__126248$fn__126251@3b5745af", "column": "name" }, { @@ -69,7 +68,6 @@ "name": "text-total-occurrences", "weight": 4, "match": "x-axis: native weekly", - "match-context-thunk": "metabase.search.scoring$text_scores_with$iter__126238__126244$fn__126245$iter__126240__126246$fn__126247$fn__126248$fn__126251@604e7000", "column": "name" }, { @@ -77,7 +75,6 @@ "name": "text-fullness", "weight": 2, "match": "x-axis: native weekly", - "match-context-thunk": "metabase.search.scoring$text_scores_with$iter__126238__126244$fn__126245$iter__126240__126246$fn__126247$fn__126248$fn__126251@2a40c731", "column": "name" } ], diff --git a/frontend/src/metabase/static-viz/components/ComboChart/stories-data/ticks-native-week-with-gap-short-range.json b/frontend/src/metabase/static-viz/components/ComboChart/stories-data/ticks-native-week-with-gap-short-range.json index fb3793d3cc701c8b1909f63d88f96bb72f569294..0515e43ee312cbc81eb90095b78ffcf0efabe483 100644 --- a/frontend/src/metabase/static-viz/components/ComboChart/stories-data/ticks-native-week-with-gap-short-range.json +++ b/frontend/src/metabase/static-viz/components/ComboChart/stories-data/ticks-native-week-with-gap-short-range.json @@ -61,7 +61,6 @@ "name": "text-consecutivity", "weight": 4, "match": "x-axis: native weekly", - "match-context-thunk": "metabase.search.scoring$text_scores_with$iter__126238__126244$fn__126245$iter__126240__126246$fn__126247$fn__126248$fn__126251@3b5745af", "column": "name" }, { @@ -69,7 +68,6 @@ "name": "text-total-occurrences", "weight": 4, "match": "x-axis: native weekly", - "match-context-thunk": "metabase.search.scoring$text_scores_with$iter__126238__126244$fn__126245$iter__126240__126246$fn__126247$fn__126248$fn__126251@604e7000", "column": "name" }, { @@ -77,7 +75,6 @@ "name": "text-fullness", "weight": 2, "match": "x-axis: native weekly", - "match-context-thunk": "metabase.search.scoring$text_scores_with$iter__126238__126244$fn__126245$iter__126240__126246$fn__126247$fn__126248$fn__126251@2a40c731", "column": "name" } ], diff --git a/frontend/src/metabase/static-viz/components/PieChart/stories-data/binned-dimension.json b/frontend/src/metabase/static-viz/components/PieChart/stories-data/binned-dimension.json index d3322c35380a69edd5d790540aaac0ea4ce17591..799ac9f05ec7a9b0c4a8d36b1b598cbeadf0eb81 100644 --- a/frontend/src/metabase/static-viz/components/PieChart/stories-data/binned-dimension.json +++ b/frontend/src/metabase/static-viz/components/PieChart/stories-data/binned-dimension.json @@ -151,7 +151,6 @@ "name": "text-exact-match", "weight": 4, "match": "Pie - Binned Numeric Dimension - Poke count by Gen", - "match-context-thunk": "metabase.search.scoring$text_scores_with$iter__133630__133636$fn__133637$iter__133632__133638$fn__133639$fn__133640$fn__133643@6e1f99ec", "column": "name" }, { @@ -159,7 +158,6 @@ "name": "text-consecutivity", "weight": 2, "match": "Pie - Binned Numeric Dimension - Poke count by Gen", - "match-context-thunk": "metabase.search.scoring$text_scores_with$iter__133630__133636$fn__133637$iter__133632__133638$fn__133639$fn__133640$fn__133643@6961ee08", "column": "name" }, { @@ -167,7 +165,6 @@ "name": "text-total-occurrences", "weight": 2, "match": "Pie - Binned Numeric Dimension - Poke count by Gen", - "match-context-thunk": "metabase.search.scoring$text_scores_with$iter__133630__133636$fn__133637$iter__133632__133638$fn__133639$fn__133640$fn__133643@6015b1d0", "column": "name" }, { @@ -175,7 +172,6 @@ "name": "text-fullness", "weight": 1, "match": "Pie - Binned Numeric Dimension - Poke count by Gen", - "match-context-thunk": "metabase.search.scoring$text_scores_with$iter__133630__133636$fn__133637$iter__133632__133638$fn__133639$fn__133640$fn__133643@3ed2f680", "column": "name" }, { @@ -183,7 +179,6 @@ "name": "text-prefix", "weight": 1, "match": "Pie - Binned Numeric Dimension - Poke count by Gen", - "match-context-thunk": "metabase.search.scoring$text_scores_with$iter__133630__133636$fn__133637$iter__133632__133638$fn__133639$fn__133640$fn__133643@2c10d84f", "column": "name" } ], diff --git a/frontend/src/metabase/static-viz/components/PieChart/stories-data/numeric-dimension.json b/frontend/src/metabase/static-viz/components/PieChart/stories-data/numeric-dimension.json index a19583b2fa0a1a65f29f75b55570431cdd9a54b7..b7bba6ae9da3f34135710a3daa15e4d1ce825ac7 100644 --- a/frontend/src/metabase/static-viz/components/PieChart/stories-data/numeric-dimension.json +++ b/frontend/src/metabase/static-viz/components/PieChart/stories-data/numeric-dimension.json @@ -144,7 +144,6 @@ "name": "text-exact-match", "weight": 4, "match": "Pie - Linear/Numeric Dimension - Poke count by Gen", - "match-context-thunk": "metabase.search.scoring$text_scores_with$iter__133630__133636$fn__133637$iter__133632__133638$fn__133639$fn__133640$fn__133643@77a87a5f", "column": "name" }, { @@ -152,7 +151,6 @@ "name": "text-consecutivity", "weight": 2, "match": "Pie - Linear/Numeric Dimension - Poke count by Gen", - "match-context-thunk": "metabase.search.scoring$text_scores_with$iter__133630__133636$fn__133637$iter__133632__133638$fn__133639$fn__133640$fn__133643@5b61d4b4", "column": "name" }, { @@ -160,7 +158,6 @@ "name": "text-total-occurrences", "weight": 2, "match": "Pie - Linear/Numeric Dimension - Poke count by Gen", - "match-context-thunk": "metabase.search.scoring$text_scores_with$iter__133630__133636$fn__133637$iter__133632__133638$fn__133639$fn__133640$fn__133643@681de957", "column": "name" }, { @@ -168,7 +165,6 @@ "name": "text-fullness", "weight": 1, "match": "Pie - Linear/Numeric Dimension - Poke count by Gen", - "match-context-thunk": "metabase.search.scoring$text_scores_with$iter__133630__133636$fn__133637$iter__133632__133638$fn__133639$fn__133640$fn__133643@7118b132", "column": "name" }, { @@ -176,7 +172,6 @@ "name": "text-prefix", "weight": 1, "match": "Pie - Linear/Numeric Dimension - Poke count by Gen", - "match-context-thunk": "metabase.search.scoring$text_scores_with$iter__133630__133636$fn__133637$iter__133632__133638$fn__133639$fn__133640$fn__133643@455657cc", "column": "name" } ], diff --git a/frontend/src/metabase/static-viz/components/WaterfallChart/stories-data/linear-null-dimension.json b/frontend/src/metabase/static-viz/components/WaterfallChart/stories-data/linear-null-dimension.json index 1ab110fb046a1bed343e83bd8f5af73c395ded59..1c69144b1a8058c7b0b126e8378798601907b51c 100644 --- a/frontend/src/metabase/static-viz/components/WaterfallChart/stories-data/linear-null-dimension.json +++ b/frontend/src/metabase/static-viz/components/WaterfallChart/stories-data/linear-null-dimension.json @@ -35,7 +35,6 @@ "name": "text-exact-match", "weight": 4.444444444444444, "match": "Waterfall Linear Null Dimension", - "match-context-thunk": "metabase.search.scoring$text_scores_with$iter__126238__126244$fn__126245$iter__126240__126246$fn__126247$fn__126248$fn__126251@3cd854d4", "column": "name" }, { @@ -43,7 +42,6 @@ "name": "text-consecutivity", "weight": 2.222222222222222, "match": "Waterfall Linear Null Dimension", - "match-context-thunk": "metabase.search.scoring$text_scores_with$iter__126238__126244$fn__126245$iter__126240__126246$fn__126247$fn__126248$fn__126251@189ff8e6", "column": "name" }, { @@ -51,7 +49,6 @@ "name": "text-total-occurrences", "weight": 2.222222222222222, "match": "Waterfall Linear Null Dimension", - "match-context-thunk": "metabase.search.scoring$text_scores_with$iter__126238__126244$fn__126245$iter__126240__126246$fn__126247$fn__126248$fn__126251@4986c53e", "column": "name" }, { @@ -59,7 +56,6 @@ "name": "text-fullness", "weight": 1.111111111111111, "match": "Waterfall Linear Null Dimension", - "match-context-thunk": "metabase.search.scoring$text_scores_with$iter__126238__126244$fn__126245$iter__126240__126246$fn__126247$fn__126248$fn__126251@70414c19", "column": "name" } ], diff --git a/frontend/src/metabase/static-viz/components/WaterfallChart/stories-data/native-time-series-quarter.json b/frontend/src/metabase/static-viz/components/WaterfallChart/stories-data/native-time-series-quarter.json index 064d26ea3a31c88f77136e6f864575dbcf7bc016..81c6ea9175f5000aa1fe78a0fcf17b11654e1eed 100644 --- a/frontend/src/metabase/static-viz/components/WaterfallChart/stories-data/native-time-series-quarter.json +++ b/frontend/src/metabase/static-viz/components/WaterfallChart/stories-data/native-time-series-quarter.json @@ -36,7 +36,6 @@ "name": "text-exact-match", "weight": 4, "match": "waterfall quarter bucket native", - "match-context-thunk": "metabase.search.scoring$text_scores_with$iter__126238__126244$fn__126245$iter__126240__126246$fn__126247$fn__126248$fn__126251@1310237b", "column": "name" }, { @@ -44,7 +43,6 @@ "name": "text-consecutivity", "weight": 2, "match": "waterfall quarter bucket native", - "match-context-thunk": "metabase.search.scoring$text_scores_with$iter__126238__126244$fn__126245$iter__126240__126246$fn__126247$fn__126248$fn__126251@4b65a3ce", "column": "name" }, { @@ -52,7 +50,6 @@ "name": "text-total-occurrences", "weight": 2, "match": "waterfall quarter bucket native", - "match-context-thunk": "metabase.search.scoring$text_scores_with$iter__126238__126244$fn__126245$iter__126240__126246$fn__126247$fn__126248$fn__126251@794e8bd7", "column": "name" }, { @@ -60,7 +57,6 @@ "name": "text-fullness", "weight": 1, "match": "waterfall quarter bucket native", - "match-context-thunk": "metabase.search.scoring$text_scores_with$iter__126238__126244$fn__126245$iter__126240__126246$fn__126247$fn__126248$fn__126251@4a0c822f", "column": "name" }, { @@ -68,7 +64,6 @@ "name": "text-prefix", "weight": 1, "match": "waterfall quarter bucket native", - "match-context-thunk": "metabase.search.scoring$text_scores_with$iter__126238__126244$fn__126245$iter__126240__126246$fn__126247$fn__126248$fn__126251@58957bee", "column": "name" } ], diff --git a/frontend/src/metabase/static-viz/components/WaterfallChart/stories-data/ordinal-null-dimension.json b/frontend/src/metabase/static-viz/components/WaterfallChart/stories-data/ordinal-null-dimension.json index b3e4673cc6f58005f661da7edef652889d91047c..9878a5b79492472c0ec107189f126e02adb22f34 100644 --- a/frontend/src/metabase/static-viz/components/WaterfallChart/stories-data/ordinal-null-dimension.json +++ b/frontend/src/metabase/static-viz/components/WaterfallChart/stories-data/ordinal-null-dimension.json @@ -154,7 +154,6 @@ "name": "text-consecutivity", "weight": 3.333333333333333, "match": "Waterfall Ordinal Null Dimension", - "match-context-thunk": "metabase.search.scoring$text_scores_with$iter__126238__126244$fn__126245$iter__126240__126246$fn__126247$fn__126248$fn__126251@6385cf53", "column": "name" }, { @@ -162,7 +161,6 @@ "name": "text-total-occurrences", "weight": 3.333333333333333, "match": "Waterfall Ordinal Null Dimension", - "match-context-thunk": "metabase.search.scoring$text_scores_with$iter__126238__126244$fn__126245$iter__126240__126246$fn__126247$fn__126248$fn__126251@8bbd715", "column": "name" }, { @@ -170,7 +168,6 @@ "name": "text-fullness", "weight": 1.666666666666667, "match": "Waterfall Ordinal Null Dimension", - "match-context-thunk": "metabase.search.scoring$text_scores_with$iter__126238__126244$fn__126245$iter__126240__126246$fn__126247$fn__126248$fn__126251@1592f629", "column": "name" }, { @@ -178,7 +175,6 @@ "name": "text-prefix", "weight": 1.666666666666667, "match": "Waterfall Ordinal Null Dimension", - "match-context-thunk": "metabase.search.scoring$text_scores_with$iter__126238__126244$fn__126245$iter__126240__126246$fn__126247$fn__126248$fn__126251@37134f79", "column": "name" } ], diff --git a/frontend/src/metabase/static-viz/components/WaterfallChart/stories-data/structured-time-series-year.json b/frontend/src/metabase/static-viz/components/WaterfallChart/stories-data/structured-time-series-year.json index a3766dec2dab5de42c896e9099320121a93cb489..e65e1cc6830d0085e2318607116f74e477601b7e 100644 --- a/frontend/src/metabase/static-viz/components/WaterfallChart/stories-data/structured-time-series-year.json +++ b/frontend/src/metabase/static-viz/components/WaterfallChart/stories-data/structured-time-series-year.json @@ -35,7 +35,6 @@ "name": "text-exact-match", "weight": 4, "match": "waterfall year bucket", - "match-context-thunk": "metabase.search.scoring$text_scores_with$iter__126238__126244$fn__126245$iter__126240__126246$fn__126247$fn__126248$fn__126251@3e08c8e1", "column": "name" }, { @@ -43,7 +42,6 @@ "name": "text-consecutivity", "weight": 2, "match": "waterfall year bucket", - "match-context-thunk": "metabase.search.scoring$text_scores_with$iter__126238__126244$fn__126245$iter__126240__126246$fn__126247$fn__126248$fn__126251@3395ecc4", "column": "name" }, { @@ -51,7 +49,6 @@ "name": "text-total-occurrences", "weight": 2, "match": "waterfall year bucket", - "match-context-thunk": "metabase.search.scoring$text_scores_with$iter__126238__126244$fn__126245$iter__126240__126246$fn__126247$fn__126248$fn__126251@1da61d63", "column": "name" }, { @@ -59,7 +56,6 @@ "name": "text-fullness", "weight": 1, "match": "waterfall year bucket", - "match-context-thunk": "metabase.search.scoring$text_scores_with$iter__126238__126244$fn__126245$iter__126240__126246$fn__126247$fn__126248$fn__126251@6e9f4f32", "column": "name" }, { @@ -67,7 +63,6 @@ "name": "text-prefix", "weight": 1, "match": "waterfall year bucket", - "match-context-thunk": "metabase.search.scoring$text_scores_with$iter__126238__126244$fn__126245$iter__126240__126246$fn__126247$fn__126248$fn__126251@16a8ddc0", "column": "name" } ], diff --git a/src/metabase/search/impl.clj b/src/metabase/search/impl.clj index e2bd1a10ccee1dc64440242d9fc493f025c92b2d..df1c2f8ddf8168bd5a5bb707e2160bdeab3f0404 100644 --- a/src/metabase/search/impl.clj +++ b/src/metabase/search/impl.clj @@ -491,7 +491,8 @@ collection_authority_level collection_type collection_effective_ancestors effective_parent archived_directly model]}] (let [matching-columns (into #{} (remove nil? (map :column relevant-scores))) - match-context-thunk (first (keep :match-context-thunk relevant-scores))] + match-context-thunk (first (keep :match-context-thunk relevant-scores)) + remove-thunks (partial mapv #(dissoc % :match-context-thunk))] (-> result (assoc :name (if (and (contains? matching-columns :display_name) display_name) @@ -512,7 +513,7 @@ effective_parent (when collection_effective_ancestors {:effective_ancestors collection_effective_ancestors}))) - :scores all-scores) + :scores (remove-thunks all-scores)) (update :dataset_query (fn [dataset-query] (when-let [query (some-> dataset-query json/parse-string)] (if (get query "type") diff --git a/src/metabase/search/postgres/index.clj b/src/metabase/search/postgres/index.clj index 383765123005c8c1ab8bda2d34d0ebd47417d745..9c134994b57f1c2c4b57ed9b264c0ccd8f6ce4a3 100644 --- a/src/metabase/search/postgres/index.clj +++ b/src/metabase/search/postgres/index.clj @@ -2,6 +2,7 @@ (:require [clojure.string :as str] [honey.sql.helpers :as sql.helpers] + [metabase.util :as u] [toucan2.core :as t2])) (def ^:private active-table :search_index) @@ -100,15 +101,64 @@ (when @reindexing? (t2/insert! pending-table entry)))) +(defn- process-negation [term] + (if (str/starts-with? term "-") + (str "!" (subs term 1)) + term)) + +(defn- process-phrase [word-or-phrase] + ;; a phrase is quoted even if the closing quotation mark has not been typed yet + (if (str/starts-with? word-or-phrase "\"") + ;; quoted phrases must be matched sequentially + (as-> word-or-phrase <> + ;; remove the quote mark(s) + (str/replace <> #"^\"|\"$" "") + (str/trim <>) + (str/split <> #"\s+") + (str/join " <-> " <>)) + ;; just a regular word + word-or-phrase)) + +(defn- split-preserving-quotes + "Break up the words in the search input, preserving quoted and partially quoted segments." + [s] + (re-seq #"\"[^\"]*(?:\"|$)|[^\s\"]+|\s+" (u/lower-case-en s))) + +(defn- process-clause [words-and-phrases] + (->> words-and-phrases + (remove #{"and"}) + (map (comp process-phrase + process-negation)) + (str/join " & "))) + +(defn- complete-last-word + "Add wildcards at the end of the final word, so that we match ts completions." + [expression] + (str/replace expression #"(\S+)(?=\s*$)" "$1:*")) + +(defn- to-tsquery-expr + "Given the user input, construct a query in the Postgres tsvector query language." + [input] + (let [trimmed (str/trim input) + complete? (not (str/ends-with? trimmed "\"")) + maybe-complete (if complete? complete-last-word identity)] + (->> (split-preserving-quotes trimmed) + (remove str/blank?) + (partition-by #{"or"}) + (remove #(= (first %) "or")) + (map process-clause) + (str/join " | ") + maybe-complete))) + (defn search-query "Query fragment for all models corresponding to a query paramter `:search-term`." [search-term] {:select [:model_id :model] :from [active-table] :where [:raw - "search_vector @@ websearch_to_tsquery('" + "search_vector @@ to_tsquery('" tsv-language "', " - [:lift search-term] ")"]}) + [:lift (to-tsquery-expr search-term)] ")"]}) (defn search "Use the index table to search for records." diff --git a/test/metabase/search/postgres/index_test.clj b/test/metabase/search/postgres/index_test.clj index d08b2878cf7b47c81c62d7dd0844bef911d64d45..654fe9ea047cf0ecbba76848465c1542c42406b5 100644 --- a/test/metabase/search/postgres/index_test.clj +++ b/test/metabase/search/postgres/index_test.clj @@ -58,7 +58,14 @@ (doseq [[a b] [["revenue" "revenues"] ["collect" "collection"]]] (is (= (search.index/search a) - (search.index/search b))))))) + (search.index/search b))))) + + (testing "Or we match a completion of the final word" + (is (seq (search.index/search "ras"))) + (is (seq (search.index/search "rasta coll"))) + (is (seq (search.index/search "collection ras"))) + (is (empty? (search.index/search "coll rasta"))) + (is (empty? (search.index/search "ras collection")))))) (deftest either-test (with-index @@ -99,3 +106,35 @@ (testing "legacy search has a bunch of results" (is (= 3 (legacy-hits "projected revenue"))) (is (= 0 (legacy-hits "\"projected revenue\"")))))) + +;; lower level search expression tests + +(def search-expr #'search.index/to-tsquery-expr) + +(deftest to-tsquery-expr-test + (is (= "a & b & c:*" + (search-expr "a b c"))) + + (is (= "a & b & c:*" + (search-expr "a AND b AND c"))) + + (is (= "a & b & c" + (search-expr "a b \"c\""))) + + (is (= "a & b | c:*" + (search-expr "a b or c"))) + + (is (= "this & !that:*" + (search-expr "this -that"))) + + (is (= "a & b & c <-> d & e | b & e:*" + (search-expr "a b \" c d\" e or b e"))) + + (is (= "ab <-> and <-> cde <-> f | !abc & def & ghi | jkl <-> mno <-> or <-> pqr" + (search-expr "\"ab and cde f\" or -abc def AND ghi OR \"jkl mno OR pqr\""))) + + (is (= "big & data | business <-> intelligence | data & wrangling:*" + (search-expr "Big Data oR \"Business Intelligence\" OR data and wrangling"))) + + (is (= "partial <-> quoted <-> and <-> or <-> -split:*" + (search-expr "\"partial quoted AND OR -split"))))