Skip to content
Snippets Groups Projects
Unverified Commit c45c9be1 authored by Cam Saul's avatar Cam Saul Committed by GitHub
Browse files

Implement functions for manipulating join strategies (#31181)

parent e01bcd33
No related branches found
No related tags found
No related merge requests found
import * as ML from "cljs/metabase.lib.js";
import type {
ColumnMetadata,
FilterOperator,
Join,
JoinStrategy,
Query,
TableMetadata,
CardMetadata,
} from "./types";
export function joinStrategy(join: Join): JoinStrategy {
return ML.join_strategy(join);
}
export function withJoinStrategy(join: Join, strategy: JoinStrategy): Join {
return ML.with_join_strategy(join, strategy);
}
export function availableJoinStrategies(
query: Query,
stageIndex: number,
): JoinStrategy[] {
return ML.available_join_strategies(query, stageIndex);
}
export function joinConditionLHSColumns(
query: Query,
stageIndex: number,
rhsColumn?: ColumnMetadata,
): ColumnMetadata[] {
return ML.join_condition_lhs_columns(query, stageIndex, rhsColumn);
}
export function joinConditionRHSColumns(
query: Query,
stageIndex: number,
joinedThing: TableMetadata | CardMetadata,
lhsColumn?: ColumnMetadata,
): ColumnMetadata[] {
return ML.join_condition_rhs_columns(
query,
stageIndex,
joinedThing,
lhsColumn,
);
}
export function joinConditionOperators(
query: Query,
stageIndex: number,
lhsColumn?: ColumnMetadata,
rhsColumn?: ColumnMetadata,
): FilterOperator[] {
return ML.join_condition_operators(query, stageIndex, lhsColumn, rhsColumn);
}
......@@ -8,6 +8,12 @@ export type Query = unknown & { _opaque: typeof Query };
declare const MetadataProvider: unique symbol;
export type MetadataProvider = unknown & { _opaque: typeof MetadataProvider };
declare const TableMetadata: unique symbol;
export type TableMetadata = unknown & { _opaque: typeof TableMetadata };
declare const CardMetadata: unique symbol;
export type CardMetadata = unknown & { _opaque: typeof CardMetadata };
export type Limit = number | null;
declare const OrderByClause: unique symbol;
......@@ -55,3 +61,15 @@ export type OrderByClauseDisplayInfo = Pick<
> & {
direction: OrderByDirection;
};
declare const FilterOperator: unique symbol;
export type FilterOperator = unknown & { _opaque: typeof FilterOperator };
declare const Join: unique symbol;
export type Join = unknown & { _opaque: typeof Join };
export type JoinStrategy =
| "left-join"
| "right-join"
| "inner-join"
| "full-join";
......@@ -7,5 +7,6 @@ export * from "./breakout";
export * from "./fields";
export * from "./limit";
export * from "./order_by";
export * from "./join";
export * from "./query";
export * from "./types";
......@@ -167,7 +167,13 @@
join-fields
joins
with-join-alias
with-join-fields]
with-join-fields
join-strategy
with-join-strategy
available-join-strategies
join-condition-lhs-columns
join-condition-rhs-columns
join-condition-operators]
[lib.limit
current-limit
limit]
......
......@@ -9,7 +9,9 @@
[metabase.lib.options :as lib.options]
[metabase.lib.schema :as lib.schema]
[metabase.lib.schema.common :as lib.schema.common]
[metabase.lib.schema.filter :as lib.schema.filter]
[metabase.lib.schema.join :as lib.schema.join]
[metabase.lib.types.isa :as lib.types.isa]
[metabase.lib.util :as lib.util]
[metabase.shared.util.i18n :as i18n]
[metabase.util.malli :as mu]))
......@@ -314,3 +316,145 @@
"Get all join conditions for the given join"
[j :- ::lib.schema.join/join]
(:fields j))
(mu/defn join-strategy :- ::lib.schema.join/strategy
"Get the raw keyword strategy (type) of a given join, e.g. `:left-join` or `:right-join`. This is either the value
of the optional `:strategy` key or the default, `:left-join`, if `:strategy` is not specified."
[a-join :- ::lib.schema.join/join]
(get a-join :strategy :left-join))
(mu/defn with-join-strategy :- [:or ::lib.schema.join/join fn?]
"Return a copy of `a-join` with its `:strategy` set to `strategy`."
[a-join :- [:or
::lib.schema.join/join
fn?]
strategy :- ::lib.schema.join/strategy]
(if (fn? a-join)
(fn [query stage-metadata]
(with-join-strategy (a-join query stage-metadata) strategy))
(assoc a-join :strategy strategy)))
(mu/defn available-join-strategies :- [:sequential ::lib.schema.join/strategy]
"Get available join strategies for the current Database (based on the Database's
supported [[metabase.driver/driver-features]]) as raw keywords like `:left-join`."
([query]
(available-join-strategies query -1))
;; stage number is not currently used, but it is taken as a parameter for consistency with the rest of MLv2
([query :- ::lib.schema/query
_stage-number :- :int]
(let [database (lib.metadata/database query)
features (:features database)]
(filterv (partial contains? features)
[:left-join
:right-join
:inner-join
:full-join]))))
;;; Building join conditions:
;;;
;;; The QB GUI needs to build a join condition before the join itself is attached to the query. There are three parts
;;; to a join condition. Suppose we're building a query like
;;;
;;; SELECT * FROM order JOIN user ON order.user_id = user.id
;;;
;;; The condition is
;;;
;;; order.user_id = user.id
;;; ^^^^^^^^^^^^^ ^ ^^^^^^^
;;; 1 2 3
;;;
;;; and the three parts are:
;;;
;;; 1. LHS/source column: the column in the left-hand side of the condition, e.g. the `order.user_id` in the example
;;; above. Either comes from the source Table, or a previous stage of the query, or a previously-joined
;;; Table/Model/Saved Question. `order.user_id` presumably is an FK to `user.id`, and while this is typical, is not
;;; required.
;;;
;;; 2. The operator: `=` in the example above. Corresponds to an `:=` MBQL clause. `=` is selected by default.
;;;
;;; 3. RHS/destination/target column: the column in the right-hand side of the condition e.g. `user.id` in the example
;;; above. `user.id` is a column in the Table/Model/Saved Question we are joining against.
;;;
;;; The Query Builder allows selecting any of these three parts in any order. The functions below return possible
;;; options for each respective part. At the time of this writing, selecting one does not filter out incompatible
;;; options for the other parts, but hopefully we can implement this in the future -- see #31174
(mu/defn ^:private sort-join-condition-columns :- [:sequential lib.metadata/ColumnMetadata]
"Sort potential join condition columns as returned by [[join-condition-lhs-columns]]
or [[join-condition-rhs-columns]]. PK columns are returned first, followed by FK columns, followed by other columns.
Otherwise original order is maintained."
[columns :- [:sequential lib.metadata/ColumnMetadata]]
(let [{:keys [pk fk other]} (group-by (fn [column]
(cond
(lib.types.isa/primary-key? column) :pk
(lib.types.isa/foreign-key? column) :fk
:else :other))
columns)]
(concat pk fk other)))
(mu/defn join-condition-lhs-columns :- [:sequential lib.metadata/ColumnMetadata]
"Get a sequence of columns that can be used as the left-hand-side (source column) in a join condition. This column
is the one that comes from the source Table/Card/previous stage of the query or a previous join.
If the right-hand-side column has already been chosen (they can be chosen in any order in the Query Builder UI),
pass in the chosen RHS column. In the future, this may be used to restrict results to compatible columns. (See #31174)
Results will be returned in a 'somewhat smart' order with PKs and FKs returned before other columns.
Unlike most other things that return columns, implicitly-joinable columns ARE NOT returned here."
([query rhs-column-or-nil]
(join-condition-lhs-columns query -1 rhs-column-or-nil))
([query :- ::lib.schema/query
stage-number :- :int
;; not yet used, hopefully we will use in the future when present for filtering incompatible columns out.
_rhs-column-or-nil :- [:maybe lib.metadata/ColumnMetadata]]
(sort-join-condition-columns
(lib.metadata.calculation/visible-columns query
stage-number
(lib.util/query-stage query stage-number)
{:include-implicitly-joinable? false}))))
(mu/defn join-condition-rhs-columns :- [:sequential lib.metadata/ColumnMetadata]
"Get a sequence of columns that can be used as the right-hand-side (target column) in a join condition. This column
is the one that belongs to the thing being joined, `joined-thing`, which can be something like a
Table ([[metabase.lib.metadata/TableMetadata]]), Saved Question/Model ([[metabase.lib.metadata/CardMetadata]]),
another query, etc. -- anything you can pass to [[join-clause]].
If the lhs-hand-side column has already been chosen (they can be chosen in any order in the Query Builder UI),
pass in the chosen LHS column. In the future, this may be used to restrict results to compatible columns. (See #31174)
Results will be returned in a 'somewhat smart' order with PKs and FKs returned before other columns."
([query joined-thing lhs-column-or-nil]
(join-condition-rhs-columns query -1 joined-thing lhs-column-or-nil))
([query :- ::lib.schema/query
stage-number :- :int
joined-thing
;; not yet used, hopefully we will use in the future when present for filtering incompatible columns out.
_lhs-column-or-nil :- [:maybe lib.metadata/ColumnMetadata]]
;; I was on the fence about whether these should get `:lib/source :source/joins` or not -- it seems like based on
;; the QB UI they shouldn't. See screenshots in #31174
(sort-join-condition-columns
(lib.metadata.calculation/visible-columns query stage-number joined-thing {:include-implicitly-joinable? false}))))
(mu/defn join-condition-operators :- [:sequential ::lib.schema.filter/operator]
"Return a sequence of valid filter clause operators that can be used to build a join condition. In the Query Builder
UI, this can be chosen at any point before or after choosing the LHS and RHS. Invalid options are not currently
filtered out based on values of the LHS or RHS, but in the future we can add this -- see #31174."
([query lhs-column-or-nil rhs-column-or-nil]
(join-condition-operators query -1 lhs-column-or-nil rhs-column-or-nil))
([_query :- ::lib.schema/query
_stage-number :- :int
;; not yet used, hopefully we will use in the future when present for filtering incompatible options out.
_lhs-column-or-nil :- [:maybe lib.metadata/ColumnMetadata]
_rhs-column-or-nil :- [:maybe lib.metadata/ColumnMetadata]]
;; currently hardcoded to these six operators regardless of LHS and RHS.
[{:lib/type :mbql.filter/operator, :short :=, :display-name (i18n/tru "Equal to")}
{:lib/type :mbql.filter/operator, :short :>, :display-name (i18n/tru "Greater than")}
{:lib/type :mbql.filter/operator, :short :<, :display-name (i18n/tru "Less than")}
{:lib/type :mbql.filter/operator, :short :>=, :display-name (i18n/tru "Greater than or equal to")}
{:lib/type :mbql.filter/operator, :short :<=, :display-name (i18n/tru "Less than or equal to")}
{:lib/type :mbql.filter/operator, :short :!=, :display-name (i18n/tru "Not equal to")}]))
......@@ -407,3 +407,52 @@
"Return a sequence of column metadatas for columns that you can specify in the `:fields` of a query."
[a-query stage-number]
(to-array (lib.core/fieldable-columns a-query stage-number)))
(defn ^:export join-strategy
"Get the strategy (type) of a given join as a plain string like `left-join`."
[a-join]
(u/qualified-name (lib.core/join-strategy a-join)))
(defn ^:export with-join-strategy
"Return a copy of `a-join` with its `:strategy` set to `strategy`."
[a-join strategy]
(lib.core/with-join-strategy a-join (keyword strategy)))
(defn ^:export available-join-strategies
"Get available join strategies for the current Database (based on the Database's
supported [[metabase.driver/driver-features]]) as strings like `left-join`."
[a-query stage-number]
(to-array (map u/qualified-name (lib.core/available-join-strategies a-query stage-number))))
(defn ^:export join-condition-lhs-columns
"Get a sequence of columns that can be used as the left-hand-side (source column) in a join condition. This column
is the one that comes from the source Table/Card/previous stage of the query or a previous join.
If the right-hand-side column has already been chosen (they can be chosen in any order in the Query Builder UI),
pass in the chosen RHS column. In the future, this may be used to restrict results to compatible columns. (See #31174)
Results will be returned in a 'somewhat smart' order with PKs and FKs returned before other columns.
Unlike most other things that return columns, implicitly-joinable columns ARE NOT returned here."
[a-query stage-number rhs-column-or-nil]
(to-array (lib.core/join-condition-lhs-columns a-query stage-number rhs-column-or-nil)))
(defn ^:export join-condition-rhs-columns
"Get a sequence of columns that can be used as the right-hand-side (target column) in a join condition. This column
is the one that belongs to the thing being joined, `joined-thing`, which can be something like a
Table ([[metabase.lib.metadata/TableMetadata]]), Saved Question/Model ([[metabase.lib.metadata/CardMetadata]]),
another query, etc. -- anything you can pass to [[join-clause]].
If the lhs-hand-side column has already been chosen (they can be chosen in any order in the Query Builder UI),
pass in the chosen LHS column. In the future, this may be used to restrict results to compatible columns. (See #31174)
Results will be returned in a 'somewhat smart' order with PKs and FKs returned before other columns."
[a-query stage-number joined-thing lhs-column-or-nil]
(to-array (lib.core/join-condition-rhs-columns a-query stage-number joined-thing lhs-column-or-nil)))
(defn ^:export join-condition-operators
"Return a sequence of valid filter clause operators that can be used to build a join condition. In the Query Builder
UI, this can be chosen at any point before or after choosing the LHS and RHS. Invalid options are not currently
filtered out based on values of the LHS or RHS, but in the future we can add this -- see #31174."
[a-query stage-number lhs-column-or-nil rhs-column-or-nil]
(to-array (lib.core/join-condition-operators a-query stage-number lhs-column-or-nil rhs-column-or-nil)))
......@@ -100,11 +100,12 @@
;; for [[metabase.lib.field/fieldable-columns]] it means its already present in `:fields`
[:selected? {:optional true} :boolean]])
(def ^:private CardMetadata
"More or less the same as a [[metabase.models.card]], but with kebab-case keys. Note that the `:dataset-query` is not
necessarily converted to pMBQL yet. Probably safe to assume it is normalized however. Likewise, `:result-metadata`
is probably not quite massaged into a sequence of `ColumnMetadata`s just yet.
See [[metabase.lib.card/card-metadata-columns]] that converts these as needed."
(def CardMetadata
"Schema for metadata about a specific Saved Question (which may or may not be a Model). More or less the same as
a [[metabase.models.card]], but with kebab-case keys. Note that the `:dataset-query` is not necessarily converted to
pMBQL yet. Probably safe to assume it is normalized however. Likewise, `:result-metadata` is probably not quite
massaged into a sequence of `ColumnMetadata`s just yet. See [[metabase.lib.card/card-metadata-columns]] that
converts these as needed."
[:map
[:lib/type [:= :metadata/card]]
[:id ::lib.schema.id/card]
......@@ -130,8 +131,9 @@
[:id ::lib.schema.id/metric]
[:name ::lib.schema.common/non-blank-string]])
(def ^:private TableMetadata
"More or less the same as a [[metabase.models.table]], but with kebab-case keys."
(def TableMetadata
"Schema for metadata about a specific [[metabase.models.table]]. More or less the same as a [[metabase.models.table]],
but with kebab-case keys."
[:map
[:lib/type [:= :metadata/table]]
[:id ::lib.schema.id/table]
......@@ -146,7 +148,10 @@
[:lib/type [:= :metadata/database]]
[:id ::lib.schema.id/database]
;; Like `:fields` for [[TableMetadata]], this is now optional -- we can fetch the Tables separately if needed.
[:tables {:optional true} [:sequential TableMetadata]]])
[:tables {:optional true} [:sequential TableMetadata]]
;; TODO -- this should validate against the driver features list in [[metabase.driver/driver-features]] if we're in
;; Clj mode
[:features {:optional true} [:set :keyword]]])
(def MetadataProvider
"Schema for something that satisfies the [[lib.metadata.protocols/MetadataProvider]] protocol."
......
......@@ -40,6 +40,18 @@
(mr/def ::conditions
[:sequential {:min 1} [:ref ::expression/boolean]])
;;; valid values for the optional `:strategy` key in a join. Note that these are only valid if the current Database
;;; supports that specific join type; these match 1:1 with the Database `:features`, e.g. a Database that supports
;;; left joins will support the `:left-join` feature.
;;;
;;; When `:strategy` is not specified, `:left-join` is the default strategy.
(mr/def ::strategy
[:enum
:left-join
:right-join
:inner-join
:full-join])
(mr/def ::join
[:map
[:lib/type [:= :mbql/join]]
......@@ -47,7 +59,8 @@
[:stages [:ref :metabase.lib.schema/stages]]
[:conditions ::conditions]
[:fields {:optional true} ::fields]
[:alias {:optional true} ::alias]])
[:alias {:optional true} ::alias]
[:strategy {:optional true} ::strategy]])
(mr/def ::joins
[:and
......
......@@ -13,7 +13,7 @@
#?(:cljs (comment metabase.test-runner.assert-exprs.approximately-equal/keep-me))
(deftest ^:parallel resolve-join-test
(let [query (lib/query meta/metadata-provider (meta/table-metadata :venues))
(let [query lib.tu/venues-query
join-clause (-> ((lib/join-clause
(meta/table-metadata :categories)
[(lib/=
......@@ -275,3 +275,106 @@
:lib/source-column-alias "ID"
:lib/desired-column-alias "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXY_bfaf4e7b"}]
(lib.metadata.calculation/metadata query)))))
(deftest ^:parallel join-strategy-test
(let [query (lib.tu/query-with-join)
[join] (lib/joins query)]
(testing "join without :strategy"
(is (= :left-join
(lib/join-strategy join))))
(testing "join with explicit :strategy"
(let [join' (lib/with-join-strategy join :right-join)]
(is (=? {:strategy :right-join}
join'))
(is (= :right-join
(lib/join-strategy join')))))))
(deftest ^:parallel with-join-strategy-test
(testing "Make sure `with-join-alias` works with unresolved functions"
(is (=? {:stages [{:joins [{:strategy :right-join}]}]}
(-> lib.tu/venues-query
(lib/join (-> (lib/join-clause (fn [_query _stage-number]
(meta/table-metadata :categories))
[(lib/=
(lib/field "VENUES" "CATEGORY_ID")
(lib/with-join-alias (lib/field "CATEGORIES" "ID") "Cat"))])
(lib/with-join-strategy :right-join))))))))
(deftest ^:parallel available-join-strategies-test
(is (= [:left-join :right-join :inner-join]
(lib/available-join-strategies (lib.tu/query-with-join)))))
(defn- query-with-join-with-fields
"A query against `VENUES` joining `CATEGORIES` with `:fields` set to return only `NAME`."
[]
(-> lib.tu/venues-query
(lib/join (-> (lib/join-clause
(meta/table-metadata :categories)
[(lib/=
(lib/field "VENUES" "CATEGORY_ID")
(lib/with-join-alias (lib/field "CATEGORIES" "ID") "Cat"))])
(lib/with-join-alias "Cat")
(lib/with-join-fields [(lib/with-join-alias (lib/field "CATEGORIES" "NAME") "Cat")])))))
(deftest ^:parallel join-condition-lhs-columns-test
(let [query lib.tu/venues-query]
(doseq [rhs [nil (lib/with-join-alias (lib.metadata/field query (meta/id :venues :category-id)) "Cat")]]
(testing (str "rhs = " (pr-str rhs))
;; sort PKs then FKs then everything else
(is (=? [{:lib/desired-column-alias "ID"}
{:lib/desired-column-alias "CATEGORY_ID"}
{:lib/desired-column-alias "NAME"}
{:lib/desired-column-alias "LATITUDE"}
{:lib/desired-column-alias "LONGITUDE"}
{:lib/desired-column-alias "PRICE"}]
(lib/join-condition-lhs-columns query rhs)))))))
(deftest ^:parallel join-condition-lhs-columns-with-previous-join-test
(testing "Include columns from previous join(s)"
(let [query (query-with-join-with-fields)]
(doseq [rhs [nil (lib/with-join-alias (lib.metadata/field query (meta/id :users :id)) "User")]]
(testing (str "rhs = " (pr-str rhs))
(is (=? [{:lib/desired-column-alias "ID"}
{:lib/desired-column-alias "Cat__ID"} ;; FIXME #31233
{:lib/desired-column-alias "CATEGORY_ID"}
{:lib/desired-column-alias "NAME"}
{:lib/desired-column-alias "LATITUDE"}
{:lib/desired-column-alias "LONGITUDE"}
{:lib/desired-column-alias "PRICE"}
{:lib/desired-column-alias "Cat__NAME"}]
(lib/join-condition-lhs-columns query rhs)))
(is (= (lib/join-condition-lhs-columns query rhs)
(lib/join-condition-lhs-columns query -1 rhs))))))))
(deftest ^:parallel join-condition-rhs-columns-test
(let [query lib.tu/venues-query]
(doseq [lhs [nil (lib.metadata/field query (meta/id :venues :id))]
joined-thing [(meta/table-metadata :venues)
meta/saved-question-CardMetadata]]
(testing (str "lhs = " (pr-str lhs)
"\njoined-thing = " (pr-str joined-thing))
(is (=? [{:lib/desired-column-alias "ID"}
{:lib/desired-column-alias "CATEGORY_ID"}
{:lib/desired-column-alias "NAME"}
{:lib/desired-column-alias "LATITUDE"}
{:lib/desired-column-alias "LONGITUDE"}
{:lib/desired-column-alias "PRICE"}]
(map #(select-keys % [:lib/desired-column-alias])
(lib/join-condition-rhs-columns query joined-thing lhs))))))))
(deftest ^:parallel join-condition-operators-test
;; just make sure that this doesn't barf and returns the expected output given any combination of LHS or RHS fields
;; for now until we actually implement filtering there
(let [query lib.tu/venues-query]
(doseq [lhs [nil (lib.metadata/field query (meta/id :categories :id))]
rhs [nil (lib.metadata/field query (meta/id :venues :category-id))]]
(testing (pr-str (list `lib/join-condition-operators `lib.tu/venues-query lhs rhs))
(is (=? [{:short :=}
{:short :>}
{:short :<}
{:short :>=}
{:short :<=}
{:short :!=}]
(lib/join-condition-operators lib.tu/venues-query lhs rhs)))
(is (= (lib/join-condition-operators lib.tu/venues-query lhs rhs)
(lib/join-condition-operators lib.tu/venues-query -1 lhs rhs)))))))
......@@ -37,7 +37,7 @@
:name "CATEGORY_ID"
:base-type :type/Integer
:effective-type :type/Integer
:semantic-type nil}
:semantic-type :type/FK}
x)
(lib.metadata/stage-column query "CATEGORY_ID")
(lib.metadata/stage-column query -1 "CATEGORY_ID"))))
......
......@@ -2232,7 +2232,7 @@
:name "CATEGORY_ID"
:base-type :type/Integer
:effective-type :type/Integer
:semantic-type nil
:semantic-type :type/FK
:fingerprint {:global {:distinct-count 28, :nil% 0.0}
:type {:type/Number
{:min 2.0
......@@ -2298,10 +2298,16 @@
:columns card-results-metadata})
(def saved-question
"An representative Saved Question, with [[results-metadata]]. For testing queries that use a Saved Question as their
source. If you added `:lib/type`, `:id`, and `:name`, you could also use this as
a [[metabase.lib.metadata/CardMetadata]]."
"An representative Saved Question, with [[results-metadata]], against `VENUES`. For testing queries that use a Saved
Question as their source. See also [[saved-question-CardMetadata]] below."
{:dataset-query {:database (id)
:type :query
:query {:source-table (id :venues)}}
:result-metadata card-results-metadata})
(def saved-question-CardMetadata
"Mock [[metabase.lib.metadata/CardMetadata]] with a query against `VENUES`."
(assoc saved-question
:lib/type :metadata/card
:id 1
:name "Card 1"))
......@@ -17,11 +17,7 @@
(comment metabase.test-runner.assert-exprs.approximately-equal/keep-me))
(def venues-query
{:lib/type :mbql/query
:lib/metadata meta/metadata-provider
:database (meta/id)
:stages [{:lib/type :mbql.stage/mbql
:source-table (meta/id :venues)}]})
(lib/query meta/metadata-provider (meta/table-metadata :venues)))
(defn venues-query-with-last-stage [m]
(let [query (update-in venues-query [:stages 0] merge m)]
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment