Handle native query find-and-replace for queries with card refs (#42939)

* Handle native query find-and-replace for queries with card refs

[Fixes #42582]

c.f. limitiations in #42938

* wee cleanup

src/metabase/native_query_analyzer/replacement.clj

@@ -3,7 +3,10 @@
    [clojure.string :as str]
    [macaw.core :as macaw]
    [metabase.driver.common.parameters :as params]
-   [metabase.driver.common.parameters.parse :as params.parse]))
+   [metabase.driver.common.parameters.parse :as params.parse]
+   [metabase.driver.common.parameters.values :as params.values]
+   [metabase.query-processor.setup :as qp.setup]
+   [metabase.query-processor.store :as qp.store]))
 
 (defn- first-unique
   [raw-query f]
@@ -14,14 +17,18 @@
   []
   (format "'%s'" (gensym "metabase_sentinel_")))
 
-(defn- sentinel-variable
+(defn- gen-variable-sentinel
   [raw-query]
   (first-unique raw-query gensymed-string))
 
-(defn- sentinel-field-filter
+(defn- gen-field-filter-sentinel
   [raw-query]
   (first-unique raw-query #(apply format "(%s = %s)" (repeat 2 (gensymed-string)))))
 
+(defn- gen-table-sentinel
+  [raw-query]
+  (first-unique raw-query #(str (gensym "metabase_sentinel_table_"))))
+
 (defn- braceify
   [s]
   (format "{{%s}}" s))
@@ -31,7 +38,7 @@
   (assoc all-subs new-sub (braceify (:k token))))
 
 (defn- parse-tree->clean-query
-  [raw-query tokens]
+  [raw-query tokens param->value]
   (loop
       [[token & rest] tokens
        query-so-far   ""
@@ -44,21 +51,33 @@
       (string? token)
       (recur rest (str query-so-far token) substitutions)
 
-      (params/Param? token)
-      (let [sub (sentinel-variable raw-query)]
-        (recur rest
-               (str query-so-far sub)
-               (add-tag substitutions sub token)))
+      :else
+      (let [v         (param->value (:k token))
+            card-ref? (params/ReferencedCardQuery? v)]
+        (cond
+          card-ref?
+          (let [sub (gen-table-sentinel raw-query)]
+            (recur rest
+                   (str query-so-far sub)
+                   (add-tag substitutions sub token)))
+
+          ;; Plain variable
+          (and (params/Param? token)
+               (not card-ref?))
+          (let [sub (gen-variable-sentinel raw-query)]
+            (recur rest
+                   (str query-so-far sub)
+                   (add-tag substitutions sub token)))
 
-      (params/FieldFilter? token)
-      (let [sub (sentinel-field-filter raw-query)]
-        (recur rest
-               (str query-so-far sub)
-               (add-tag substitutions sub token)))
+          (params/FieldFilter? token)
+          (let [sub (gen-field-filter-sentinel raw-query)]
+            (recur rest
+                   (str query-so-far sub)
+                   (add-tag substitutions sub token)))
 
-      :else
-      ;; will be addressed by #42582, etc.
-      (throw (ex-info "Unsupported token in native query" {:token token})))))
+          :else
+          ;; will be addressed by #42582, etc.
+          (throw (ex-info "Unsupported token in native query" {:token token})))))))
 
 (defn- replace-all
   "Return `the-string` with all the keys of `replacements` replaced by their values.
@@ -85,13 +104,21 @@
                   (update-keys clean-table)
                   (update-vals clean-table))}))
 
+(defn- param-values
+  [query]
+  (if (qp.store/initialized?)
+    (params.values/query->params-map (:native query))
+    (qp.setup/with-qp-setup [q query]
+      (params.values/query->params-map (:native q)))))
+
 (defn replace-names
   "Given a dataset_query and a map of renames (with keys `:tables` and `:columns`, as in Macaw), return a new inner query
   with the appropriate replacements made."
   [query renames]
-  (let [raw-query    (get-in query [:native :query])
-        parsed-query (params.parse/parse raw-query)
+  (let [raw-query                    (get-in query [:native :query])
+        parsed-query                 (params.parse/parse raw-query)
+        param->value                 (param-values query)
         {clean-query :query
-         tt-subs     :substitutions} (parse-tree->clean-query raw-query parsed-query)
-        renamed-query (macaw/replace-names clean-query (clean-renames-macaw-issue-32 renames))]
+         tt-subs     :substitutions} (parse-tree->clean-query raw-query parsed-query param->value)
+        renamed-query                (macaw/replace-names clean-query (clean-renames-macaw-issue-32 renames))]
     (replace-all renamed-query tt-subs)))

test/metabase/native_query_analyzer/replacement_test.clj

 (ns metabase.native-query-analyzer.replacement-test
   (:require
    [clojure.test :refer :all]
+   [metabase.lib.native :as lib-native]
    [metabase.native-query-analyzer.replacement :refer [replace-names]]
-   [metabase.test :as mt]))
+   [metabase.test :as mt]
+   [toucan2.tools.with-temp :as t2.with-temp]))
 
 (defn- q
   "Make a native query from the concatenation of the args"
   [& args]
-  (mt/native-query {:query (apply str args)}))
+  (let [query (apply str args)]
+    (mt/native-query {:query         query
+                      :template-tags (lib-native/extract-template-tags query)})))
 
 (deftest ^:parallel replace-names-simple-test
   (testing "columns can be renamed"
@@ -20,7 +24,7 @@
     (is (= "select cost, tax from purchases"
            (replace-names (q "select amount, fee from orders") {:columns {"amount" "cost"
                                                                           "fee"    "tax"}
-                                                              :tables {"orders" "purchases"}})) )))
+                                                                :tables {"orders" "purchases"}})) )))
 
 (deftest ^:parallel replace-names-whitespace-test
   (testing "comments, whitespace, etc. are preserved"
@@ -44,10 +48,20 @@
 
 (deftest ^:parallel field-filter-test
   (testing "with variables *and* field filters"
-    (is (= (str "\n\nSELECT *\nFROM folk\nWHERE\n  referral = {{source}}\n   OR \n  id = {{id}} \n  OR \n"
+    (is (= (str "SELECT *\nFROM folk\nWHERE\n  referral = {{source}}\n   OR \n  id = {{id}} \n  OR \n"
                 "  {{birthday}}\n  OR\n  {{zipcode}}\n  OR\n  {{city}} ")
-           (replace-names (q "\n\nSELECT *\nFROM people\nWHERE\n  source = {{source}}\n   OR \n  id = {{id}} \n  OR \n"
+           (replace-names (q "SELECT *\nFROM people\nWHERE\n  source = {{source}}\n   OR \n  id = {{id}} \n  OR \n"
                              "  {{birthday}}\n  OR\n  {{zipcode}}\n  OR\n  {{city}} ")
                           {:columns {"source" "referral"
                                      "city"   "town"}  ; make sure FFs aren't replaced
                            :tables  {"people" "folk"}})))))
+
+(deftest ^:parallel referenced-card-test
+  (testing "With a reference to a card"
+    (t2.with-temp/with-temp
+      [:model/Card {card-id :id} {:type          :model
+                                  :dataset_query (mt/native-query {:query "SELECT TOTAL, TAX FROM ORDERS"})}]
+      (is (= (format "SELECT SUBTOTAL FROM {{#%s}} LIMIT 3" card-id)
+             (replace-names (q (format "SELECT TOTAL FROM {{#%s}} LIMIT 3" card-id))
+                            {:columns {"TOTAL" "SUBTOTAL"}
+                             :tables  {"ORDERS" "PURCHASES"}}))))))