Fix source queries that contain expressions (#10392)

* Tests to make sure you can use source queries that contain expressions
[ci drivers]

* Simplification of QP debugging fns

* Fix source queries w/ expressions (Bump MBQL -> 1.3.3)
[ci drivers]

project.clj

@@ -99,7 +99,7 @@
                  com.sun.jmx/jmxri]]
    [medley "1.2.0"]                                                   ; lightweight lib of useful functions
    [metabase/connection-pool "1.0.2"]                                 ; simple wrapper around C3P0. JDBC connection pools
-   [metabase/mbql "1.3.1"]                                            ; MBQL language schema & util fns
+   [metabase/mbql "1.3.3"]                                            ; MBQL language schema & util fns
    [metabase/throttle "1.0.1"]                                        ; Tools for throttling access to API endpoints and other code pathways
    [javax.xml.bind/jaxb-api "2.4.0-b180830.0359"]                     ; add the `javax.xml.bind` classes which we're still using but were removed in Java 11
    [net.sf.cssbox/cssbox "4.12" :exclusions [org.slf4j/slf4j-api]]    ; HTML / CSS rendering

src/metabase/query_processor/debug.clj

@@ -12,10 +12,6 @@
             [metabase.util :as u]
             [schema.core :as s]))
 
-(def ^:private ^:dynamic *timeout*
-  "Timeout (in milliseconds) for async middleware to return a response."
-  5000)
-
 
 ;;; +----------------------------------------------------------------------------------------------------------------+
 ;;; |                                       Generic Middleware Debugging Utils                                       |
@@ -65,44 +61,42 @@
           e exception)))))
 
 (defn- debug-async
-  [{:keys [pre post exception]} qp middleware before-query respond raise & args]
-  (let [after-query        (promise)
-        before-result      (promise)
-        middleware-qp-args (promise)
+  [{:keys [pre post exception]} orig-qp middleware before-query orig-respond orig-raise & args]
+  (let [ex-context
+        {:query {:before before-query, :after (promise)}, :result (promise)}
+
+        ;; the basic idea is to pass `identity` to the middleware instead of `orig-respond`, then we can separate out
+        ;; the changes the middleware makes to the response. `mw-respond` is the `respond` function the middleware
+        ;; returns
+        wrapped-respond
+        (fn [mw-respond]
+          (fn [before-result]
+            (let [after-result (try
+                                 (mw-respond before-result)
+                                 (catch Throwable e
+                                   (rethrow "Middleware threw Exception during post-processing." ex-context e exception)))]
+              (deliver (:result ex-context) after-result)
+              (when post
+                (try
+                  (post before-result after-result)
+                  (catch Throwable e
+                    (rethrow "Error in debugging 'post' fn" ex-context e exception))))
+              (orig-respond after-result))))
+
+        wrapped-qp
+        (fn [after-query mw-respond mw-raise & args]
+          (deliver (get-in ex-context [:query :after]) after-query)
+          (when pre
+            (try
+              (pre before-query after-query)
+              (catch Throwable e
+                (rethrow "Error in debugging 'pre' fn" ex-context e exception))))
+          (apply orig-qp after-query (wrapped-respond mw-respond) mw-raise args))
 
         wrapped-raise
         (fn [e]
-          (rethrow "Middleware raised Exception."
-            {:query {:before before-query, :after after-query}, :result before-result}
-            e exception raise))
-
-        placeholder-qp
-        (fn [& args] (deliver middleware-qp-args args))
-
-        _
-        (try
-          (apply (middleware placeholder-qp) before-query identity wrapped-raise args)
-          (catch Throwable e
-            (rethrow "Middleware threw Exception during preprocessing."
-              {:query {:before before-query}}
-              e exception)))
-
-        [query mw-respond mw-raise & args] (u/deref-with-timeout middleware-qp-args *timeout*)
-
-        wrapped-respond
-        (fn [result]
-          (deliver before-result result)
-          (let [after-result (try
-                               (mw-respond result)
-                               (catch Throwable e
-                                 (rethrow "Middleware threw Exception during post-processing."
-                                   {:query {:before before-query, :after after-query}, :result result}
-                                   e exception)))]
-            (when post (post result after-result))
-            (respond after-result)))]
-    (deliver after-query query)
-    (when pre (pre before-query query))
-    (apply qp query wrapped-respond wrapped-raise args)))
+          (rethrow "Middleware raised Exception." ex-context e exception orig-raise))]
+    (apply (middleware wrapped-qp) before-query identity wrapped-raise args)))
 
 (defn- debug-with-fns
   "Wrap a `middleware` fn for debugging. `fns` is a map of functions called at various points before and after the

test/metabase/query_processor_test/nested_queries_test.clj

@@ -671,3 +671,29 @@
           :breakout     [!month.date]}
          :filter [:> *sum/Float 300]
          :limit  2}))))
+
+;; can you use nested queries that have expressions in them?
+(datasets/expect-with-drivers (qp.test/non-timeseries-drivers-with-feature :nested-queries :foreign-keys :expressions)
+  [[30] [20]]
+  (qp.test/format-rows-by [int int]
+    (qp.test/rows
+      (data/run-mbql-query venues
+        {:source-query
+         {:source-table $$venues
+          :fields       [[:expression "price-times-ten"]]
+          :expressions  {"price-times-ten" [:* $price 10]}
+          :order-by     [[:asc $id]]
+          :limit        2}}))))
+
+(datasets/expect-with-drivers (qp.test/non-timeseries-drivers-with-feature :nested-queries :foreign-keys :expressions)
+  [[30] [20]]
+  (tt/with-temp Card [{card-id :id} {:dataset_query (data/mbql-query venues
+                                                      {:fields       [[:expression "price-times-ten"]]
+                                                       :expressions  {"price-times-ten" [:* $price 10]}
+                                                       :order-by     [[:asc $id]]
+                                                       :limit        2})}]
+
+    (qp.test/format-rows-by [int int]
+      (qp.test/rows
+        (data/run-mbql-query nil
+          {:source-table (str "card__" card-id)})))))