Skip to content
Snippets Groups Projects
Commit fcf6ae33 authored by Allen Gilliland's avatar Allen Gilliland
Browse files

Merge pull request #2564 from metabase/transpile-mbql-to-native 

Allow conversion from structured MBQL queries to native queries
parents 57ef8a8e f1778efc
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Showing
with 337 additions and 296 deletions
frontend/src/card/card.controllers.js
+ 48
− 12
View file @ fcf6ae33
......@@ -103,6 +103,39 @@ CardControllers.controller('CardDetail', [
MetabaseAnalytics.trackEvent('QueryBuilder', 'Query Started', type);
}
},
// MBQL->NATVIE
// setQueryModeFn: function(type) {
// if (!card.dataset_query.type || type !== card.dataset_query.type) {
// let database = card.dataset_query.database,
// datasetQuery = angular.copy(card.dataset_query);
// // if we are going from MBQL -> Native then attempt to carry over the query
// if (type === "native") {
// let nativeQuery = createQuery("native", database).native;
// if (queryResult && queryResult.data && queryResult.data.native_form) {
// // since we have the native form from the last execution of our query, just use that
// nativeQuery = _.pick(queryResult.data.native_form, "query", "collection");
// // when the driver requires JSON we need to stringify it because it's been parsed already
// if (_.contains(["mongo", "druid"], tableMetadata.db.engine)) {
// nativeQuery.query = JSON.stringify(queryResult.data.native_form.query);
// }
// }
// // NOTE: we purposely leave the MBQL query form on the query
// datasetQuery.type = "native";
// datasetQuery.native = nativeQuery;
// // we are going from Native -> MQBL, which is only allowed if the query wasn't modified
// } else {
// datasetQuery.type = "query";
// delete datasetQuery.native;
// }
// setQuery(datasetQuery);
// MetabaseAnalytics.trackEvent('QueryBuilder', 'Query Started', type);
// }
// },
cloneCardFn: function() {
$scope.$apply(() => {
delete card.id;
......@@ -142,7 +175,6 @@ CardControllers.controller('CardDetail', [
query: null,
setQueryFn: onQueryChanged,
setDatabaseFn: setDatabase,
setTableFn: setTable, // this is used for native queries, vs. setSourceTable, which is used for MBQL
setSourceTableFn: setSourceTable,
autocompleteResultsFn: function(prefix) {
var apiCall = Metabase.db_autocomplete_suggestions({
......@@ -605,17 +637,17 @@ CardControllers.controller('CardDetail', [
newCard.dataset_query.native.query = existingQuery;
}
setCard(newCard, {runQuery: false});
// set the initial Table ID for the query if this is a native query
// set the initial collection for the query if this is a native query
// this is only used for Mongo queries which need to be ran against a specific collection
if (newCard.dataset_query.type === 'native') {
let database = _.findWhere(databases, { id: databaseId }),
tables = database ? database.tables : [],
table = tables.length > 0 ? tables[0] : null;
if (table) newCard.dataset_query.table = table.id;
if (table) newCard.dataset_query.native.collection = table.name;
}
setCard(newCard, {runQuery: false});
} else {
// if we are editing a saved query we don't want to replace the card, so just start a fresh query only
// TODO: should this clear the visualization as well?
......@@ -632,13 +664,6 @@ CardControllers.controller('CardDetail', [
return card.dataset_query;
}
/// Sets the table ID for a *native* query.
function setTable(tableID) {
let query = card.dataset_query;
query.table = tableID;
setQuery(query);
}
// This is for MBQL queries
// indicates that the table for the query should be changed to the given value
// when editing, simply update the value. otherwise, this should create a completely new card
......@@ -672,6 +697,12 @@ CardControllers.controller('CardDetail', [
delete card.description;
}
// MBQL->NATIVE
// if (dataset_query.type === "native" && dataset_query.query) {
// // if we have an old reference to an MBQL query then we can safely kill that now
// delete dataset_query.query;
// }
setQuery(dataset_query);
}
......@@ -908,6 +939,11 @@ CardControllers.controller('CardDetail', [
}
function cardIsDirty() {
// MBQL->NATVIE
// if (card.dataset_query.type === "native" && card.dataset_query.query) {
// return false;
// }
var newCardSerialized = serializeCardForUrl(card);
return newCardSerialized !== savedCardSerialized;
......
frontend/src/lib/data_grid.js
+ 2
− 1
View file @ fcf6ae33
......@@ -27,7 +27,8 @@ export function filterOnPreviewDisplay(data) {
cols: _.filter(data.cols, function(col) { return col.visibility_type !== "details-only"; }),
columns: _.map(data.cols, function(col) { return col.display_name; }),
rows: filteredRows,
rows_truncated: data.rows_truncated
rows_truncated: data.rows_truncated,
native_form: data.native_form
};
}
......
frontend/src/query_builder/NativeQueryEditor.jsx
+ 15
− 7
View file @ fcf6ae33
......@@ -23,7 +23,6 @@ export default class NativeQueryEditor extends Component {
query: PropTypes.object.isRequired,
setQueryFn: PropTypes.func.isRequired,
setDatabaseFn: PropTypes.func.isRequired,
setTableFn: PropTypes.func.isRequired,
autocompleteResultsFn: PropTypes.func.isRequired,
/// This should return an object with information about the mode the ACE Editor should use to edit the query.
/// This object should have 2 properties:
......@@ -35,7 +34,7 @@ export default class NativeQueryEditor extends Component {
componentWillMount() {
// if the sql is empty then start with the editor showing, otherwise our default is to start out collapsed
if (!this.props.query.native.query) {
if (!this.props.query.native.query || this.props.query.query) {
this.setState({
showEditor: true
});
......@@ -147,7 +146,15 @@ export default class NativeQueryEditor extends Component {
}
setTableID(tableID) {
this.props.setTableFn(tableID);
// translate the table id into the table name
let database = this.props.databases ? _.findWhere(this.props.databases, { id: this.props.query.database }) : null,
table = database ? _.findWhere(database.tables, { id: tableID }) : null;
if (table) {
let query = this.props.query;
query.native.collection = table.name;
this.setQuery(query);
}
}
render() {
......@@ -158,7 +165,7 @@ export default class NativeQueryEditor extends Component {
if (this.state.showEditor && this.props.databases && (this.props.databases.length > 1 || modeInfo.requiresTable)) {
if (this.props.databases.length > 1) {
dataSelectors.push(
<div className="GuiBuilder-section GuiBuilder-data flex align-center">
<div key="db_selector" className="GuiBuilder-section GuiBuilder-data flex align-center">
<span className="GuiBuilder-section-label Query-label">Database</span>
<DataSelector
databases={this.props.databases}
......@@ -172,17 +179,18 @@ export default class NativeQueryEditor extends Component {
let databases = this.props.databases,
dbId = this.props.query.database,
database = databases ? _.findWhere(databases, { id: dbId }) : null,
tables = database ? database.tables : [];
tables = database ? database.tables : [],
selectedTable = this.props.query.native.collection ? _.findWhere(tables, { name: this.props.query.native.collection }) : null;
dataSelectors.push(
<div className="GuiBuilder-section GuiBuilder-data flex align-center">
<div key="table_selector" className="GuiBuilder-section GuiBuilder-data flex align-center">
<span className="GuiBuilder-section-label Query-label">Table</span>
<DataSelector
ref="dataSection"
includeTables={true}
query={{
type: "query",
query: { source_table: this.props.query.table },
query: { source_table: selectedTable ? selectedTable.id : null },
database: dbId
}}
databases={[database]}
......
frontend/src/query_builder/QueryHeader.jsx
+ 33
− 1
View file @ fcf6ae33
......@@ -70,7 +70,18 @@ export default class QueryHeader extends Component {
}
onCreate(card, addToDash) {
// TODO: why are we not cleaning the card here?
// MBQL->NATIVE
// if we are a native query with an MBQL query definition, remove the old MBQL stuff (happens when going from mbql -> native)
// if (card.dataset_query.type === "native" && card.dataset_query.query) {
// delete card.dataset_query.query;
// } else if (card.dataset_query.type === "query" && card.dataset_query.native) {
// delete card.dataset_query.native;
// }
if (card.dataset_query.query) {
Query.cleanQuery(card.dataset_query.query);
}
this.requesetPromise = cancelable(this.props.cardApi.create(card).$promise);
return this.requesetPromise.then(newCard => {
this.props.notifyCardCreatedFn(newCard);
......@@ -83,6 +94,14 @@ export default class QueryHeader extends Component {
}
onSave(card, addToDash) {
// MBQL->NATIVE
// if we are a native query with an MBQL query definition, remove the old MBQL stuff (happens when going from mbql -> native)
// if (card.dataset_query.type === "native" && card.dataset_query.query) {
// delete card.dataset_query.query;
// } else if (card.dataset_query.type === "query" && card.dataset_query.native) {
// delete card.dataset_query.native;
// }
if (card.dataset_query.query) {
Query.cleanQuery(card.dataset_query.query);
}
......@@ -172,6 +191,7 @@ export default class QueryHeader extends Component {
</ModalWithTrigger>
]);
} else {
// MBQL->NATIVE
buttonSections.push([
<QueryModeToggle
key="queryModeToggle"
......@@ -306,6 +326,18 @@ export default class QueryHeader extends Component {
</a>
]);
// MBQL->NATIVE
// native mode toggle
// if (!this.props.cardIsDirtyFn()) {
// buttonSections.push([
// <QueryModeToggle
// key="queryModeToggle"
// currentQueryMode={this.props.card.dataset_query.type}
// setQueryModeFn={this.props.setQueryModeFn}
// />
// ]);
// }
return (
<ButtonBar buttons={buttonSections} className="Header-buttonSection" />
);
......
src/metabase/driver.clj
+ 26
− 18
View file @ fcf6ae33
......@@ -107,6 +107,22 @@
Is this property required? Defaults to `false`.")
(execute-query ^java.util.Map [this, ^Map query]
"Execute a query against the database and return the results.
The query passed in will contain:
{:database ^DatabaseInstance
:native {... driver specific query form such as one returned from a call to `mbql->native` ...}
:settings {:report-timezone \"US/Pacific\"
:other-setting \"and its value\"}}
Results should look like:
{:columns [\"id\", \"name\"]
:rows [[1 \"Lucky Bird\"]
[2 \"Rasta Can\"]]}")
(features ^java.util.Set [this]
"*OPTIONAL*. A set of keyword names of optional features supported by this driver, such as `:foreign-keys`. Valid features are:
......@@ -128,29 +144,21 @@
"*OPTIONAL*. Return a humanized (user-facing) version of an connection error message string.
Generic error messages are provided in the constant `connection-error-messages`; return one of these whenever possible.")
(notify-database-updated [this, ^DatabaseInstance database]
"*OPTIONAL*. Notify the driver that the attributes of the DATABASE have changed. This is specifically relevant in
the event that the driver was doing some caching or connection pooling.")
(process-native [this, {^Integer database-id :database, {^String native-query :query} :native, :as ^Map query}]
"Process a native QUERY. This function is called by `metabase.driver/process-query`.
(mbql->native ^java.util.Map [this, ^Map query]
"Transpile an MBQL structured query into the appropriate native query form.
Results should look something like:
The input query will be a fully expanded MBQL query (https://github.com/metabase/metabase/wiki/Expanded-Queries) with
all the necessary pieces of information to build a properly formatted native query for the given database.
{:columns [\"id\", \"bird_name\"]
:cols [{:name \"id\", :base_type :IntegerField}
{:name \"bird_name\", :base_type :TextField}]
:rows [[1 \"Lucky Bird\"]
[2 \"Rasta Can\"]]}")
The result of this function will be passed directly into calls to `execute-query`.
(process-mbql [this, ^Map query]
"Process a native or structured QUERY. This function is called by `metabase.driver/process-query` after performing various driver-unspecific
steps like Query Expansion and other preprocessing.
For example, a driver like Postgres would build a valid SQL expression and return a map such as:
Results should look something like:
{:query \"SELECT * FROM my_table\"}")
[{:id 1, :name \"Lucky Bird\"}
{:id 2, :name \"Rasta Can\"}]")
(notify-database-updated [this, ^DatabaseInstance database]
"*OPTIONAL*. Notify the driver that the attributes of the DATABASE have changed. This is specifically relevant in
the event that the driver was doing some caching or connection pooling.")
(process-query-in-context [this, ^IFn qp]
"*OPTIONAL*. Similar to `sync-in-context`, but for running queries rather than syncing. This should be used to do things like open DB connections
......
src/metabase/driver/bigquery.clj
+ 20
− 10
View file @ fcf6ae33
......@@ -155,9 +155,9 @@
:fields (set (table-schema->metabase-field-info (.getSchema (get-table database table-name))))})
(defn- ^QueryResponse execute-query
(defn- ^QueryResponse execute-bigquery
([{{:keys [project-id]} :details, :as database} query-string]
(execute-query (database->client database) project-id query-string))
(execute-bigquery (database->client database) project-id query-string))
([^Bigquery client, ^String project-id, ^String query-string]
{:pre [client (seq project-id) (seq query-string)]}
......@@ -224,10 +224,7 @@
;; automatically retry the query if it times out or otherwise fails. This is on top of the auto-retry added by `execute` so operations going through `process-native*` may be
;; retried up to 3 times.
(u/auto-retry 1
(post-process-native (execute-query database query-string))))
(defn- process-native [{database-id :database, {native-query :query} :native}]
(process-native* (Database database-id) native-query))
(post-process-native (execute-bigquery database query-string))))
(defn- field-values-lazy-seq [{field-name :name, :as field-instance}]
......@@ -342,12 +339,25 @@
(for [row rows]
(zipmap columns row))))
(defn- process-mbql [{{{:keys [dataset-id]} :details, :as database} :database, {{table-name :name} :source-table} :query, :as query}]
(defn- mbql->native [{{{:keys [dataset-id]} :details, :as database} :database, {{table-name :name} :source-table} :query, :as query}]
{:pre [(map? database) (seq dataset-id) (seq table-name)]}
(let [korma-form (korma-form query (entity dataset-id table-name))
sql (korma-form->sql korma-form)]
(sqlqp/log-korma-form korma-form sql)
(post-process-mbql dataset-id table-name (process-native* database sql))))
{:query sql
:table-name table-name
:mbql? true}))
(defn- execute-query [{{{:keys [dataset-id]} :details, :as database} :database, {sql :query, :keys [table-name mbql?]} :native}]
(let [results (process-native* database sql)
results (if mbql?
(post-process-mbql dataset-id table-name results)
results)
columns (vec (keys (first results)))]
{:columns columns
:rows (for [row results]
(mapv row columns))
:annotate? true}))
;; This provides an implementation of `prepare-value` that prevents korma from converting forms to prepared statement parameters (`?`)
;; TODO - Move this into `metabase.driver.generic-sql` and document it as an alternate implementation for `prepare-value` (?)
......@@ -446,6 +456,7 @@
:display-name "Auth Code"
:placeholder "4/HSk-KtxkSzTt61j5zcbee2Rmm5JHkRFbL5gD5lgkXek"
:required true}])
:execute-query (u/drop-first-arg execute-query)
;; Don't enable foreign keys when testing because BigQuery *doesn't* have a notion of foreign keys. Joins are still allowed, which puts us in a weird position, however;
;; people can manually specifiy "foreign key" relationships in admin and everything should work correctly.
;; Since we can't infer any "FK" relationships during sync our normal FK tests are not appropriate for BigQuery, so they're disabled for the time being.
......@@ -453,7 +464,6 @@
:features (constantly (when-not config/is-test?
#{:foreign-keys}))
:field-values-lazy-seq (u/drop-first-arg field-values-lazy-seq)
:process-native (u/drop-first-arg process-native)
:process-mbql (u/drop-first-arg process-mbql)}))
:mbql->native (u/drop-first-arg mbql->native)}))
(driver/register-driver! :bigquery (BigQueryDriver.))
src/metabase/driver/crate.clj
+ 5
− 6
View file @ fcf6ae33
......@@ -3,7 +3,6 @@
[korma.core :as k]
[metabase.driver :as driver]
(metabase.driver.crate [analyze :as analyze]
[native :as native]
[query-processor :as qp]
[util :as crate-util])
[metabase.driver.generic-sql :as sql]
......@@ -67,13 +66,13 @@
(u/strict-extend CrateDriver
driver/IDriver
(merge (sql/IDriverSQLDefaultsMixin)
{:details-fields (constantly [{:name "hosts"
:display-name "Hosts"
:default "//localhost:4300"}])
{:analyze-table analyze/analyze-table
:can-connect? (u/drop-first-arg can-connect?)
:date-interval crate-util/date-interval
:analyze-table analyze/analyze-table
:process-native native/process-and-run
:details-fields (constantly [{:name "hosts"
:display-name "Hosts"
:default "//localhost:4300"}])
:execute-query qp/execute-query
:features (fn [this]
(set/difference (sql/features this)
#{:foreign-keys}))})
......
src/metabase/driver/crate/native.clj deleted 100644 → 0
+ 0
− 31
View file @ 57ef8a8e
(ns metabase.driver.crate.native
(:require [clojure.java.jdbc :as jdbc]
[clojure.tools.logging :as log]
[metabase.db :refer [sel]]
[metabase.driver.generic-sql :as sql]
[metabase.driver.generic-sql.native :as n]
[metabase.models.database :refer [Database]]
[metabase.util :as u]))
(defn process-and-run
"Process and run a native (raw SQL) QUERY."
[driver {{sql :query} :native, database-id :database, :as query}]
(try (let [database (sel :one :fields [Database :engine :details] :id database-id)
db-conn (sql/db->jdbc-connection-spec database)]
(jdbc/with-db-connection [t-conn db-conn]
(let [^java.sql.Connection jdbc-connection (:connection t-conn)]
(try
;; Now run the query itself
(log/debug (u/format-color 'green "%s" sql))
(let [[columns & [first-row :as rows]] (jdbc/query t-conn sql, :as-arrays? true)]
{:rows rows
:columns columns
:cols (for [[column first-value] (partition 2 (interleave columns first-row))]
{:name column
:base_type (n/value->base-type first-value)})})))))
(catch java.sql.SQLException e
(let [^String message (or (->> (.getMessage e) ; error message comes back like 'Column "ZID" not found; SQL statement: ... [error-code]' sometimes
(re-find #"^(.*);") ; the user already knows the SQL, and error code is meaningless
second) ; so just return the part of the exception that is relevant
(.getMessage e))]
(throw (Exception. message))))))
src/metabase/driver/crate/query_processor.clj
+ 25
− 3
View file @ fcf6ae33
(ns metabase.driver.crate.query-processor
(:require [korma.core :as k]
(korma.sql [engine :as kengine]
[fns :as kfns])
(:require [clojure.java.jdbc :as jdbc]
[korma.core :as k]
[korma.sql.engine :as kengine]
[korma.sql.fns :as kfns]
[metabase.driver.generic-sql :as sql]
[metabase.driver.generic-sql.query-processor :as qp]
[metabase.query-processor.interface :as i])
(:import (metabase.query_processor.interface ComparisonFilter CompoundFilter)))
......@@ -33,3 +35,23 @@
"Apply custom generic SQL filter. This is the place to perform query rewrites."
[_ korma-form {clause :filter}]
(k/where korma-form (resolve-subclauses clause)))
(defn execute-query
"Execute a query against Crate database.
We specifically write out own `execute-query` function to avoid the autoCommit(false) call."
[_ {:keys [database], {sql :query, params :params} :native}]
(try (let [db-conn (sql/db->jdbc-connection-spec database)]
(jdbc/with-db-connection [t-conn db-conn]
(let [statement (if params
(into [sql] params)
sql)]
(let [[columns & rows] (jdbc/query t-conn statement, :identifiers identity, :as-arrays? true)]
{:rows rows
:columns columns}))))
(catch java.sql.SQLException e
(let [^String message (or (->> (.getMessage e) ; error message comes back like 'Column "ZID" not found; SQL statement: ... [error-code]' sometimes
(re-find #"^(.*);") ; the user already knows the SQL, and error code is meaningless
second) ; so just return the part of the exception that is relevant
(.getMessage e))]
(throw (Exception. message))))))
src/metabase/driver/druid.clj
+ 3
− 24
View file @ fcf6ae33
......@@ -3,14 +3,11 @@
(:require [clojure.tools.logging :as log]
[clj-http.client :as http]
[cheshire.core :as json]
[metabase.api.common :refer [let-404]]
[metabase.db :refer [sel]]
[metabase.driver :as driver]
[metabase.driver.druid.query-processor :as qp]
(metabase.models [database :refer [Database]]
[field :as field]
(metabase.models [field :as field]
[table :as table])
[metabase.query-processor.annotate :as annotate]
[metabase.util :as u]))
;;; ### Request helper fns
......@@ -66,24 +63,6 @@
(throw (Exception. message e))))))
(defn- process-mbql [query]
;; Merge `:settings` into the inner query dict so the QP has access to it
(qp/process-mbql-query (partial do-query (get-in query [:database :details]))
(assoc (:query query)
:settings (:settings query))))
(defn- process-native [{database-id :database, {query :query} :native, :as outer-query}]
{:pre [(integer? database-id) query]}
(let-404 [details (sel :one :field [Database :details], :id database-id)]
(let [query (if (string? query)
(json/parse-string query keyword)
query)]
;; `annotate` happens automatically as part of the QP middleware for MBQL queries but not for native ones.
;; This behavior was originally so we could preserve column order for raw SQL queries.
;; Since Druid results come back as maps for each row there is no order to preserve so we can go ahead and re-use the MBQL-QP annotation code here.
(annotate/annotate outer-query (qp/post-process-native query (do-query details query))))))
;;; ### Sync
(defn- describe-table-field [druid-field-type field-name]
......@@ -180,9 +159,9 @@
:display-name "Broker node port"
:type :integer
:default 8082}])
:execute-query (fn [_ query] (qp/execute-query do-query query))
:features (constantly #{:set-timezone})
:field-values-lazy-seq (u/drop-first-arg field-values-lazy-seq)
:process-native (u/drop-first-arg process-native)
:process-mbql (u/drop-first-arg process-mbql)}))
:mbql->native (u/drop-first-arg qp/mbql->native)}))
(driver/register-driver! :druid (DruidDriver.))
src/metabase/driver/druid/query_processor.clj
+ 31
− 15
View file @ fcf6ae33
......@@ -2,6 +2,7 @@
(:require [clojure.core.match :refer [match]]
[clojure.string :as s]
[clojure.tools.logging :as log]
[cheshire.core :as json]
[metabase.query-processor :as qp]
[metabase.query-processor.interface :as i]
[metabase.util :as u])
......@@ -467,10 +468,6 @@
[:many _] ::groupBy)))
(defn- log-druid-query [druid-query]
(log/debug (u/format-color 'blue "DRUID QUERY:😋\n%s\n" (u/pprint-to-str druid-query))))
(defn- build-druid-query [query]
{:pre [(map? query)]}
(let [query-type (druid-query-type query)]
......@@ -520,14 +517,33 @@
(apply dissoc result keys-to-remove)))))
;;; ### process-mbql-query
(defn process-mbql-query
"Process an MBQL (inner) query for a Druid DB."
[do-query query]
(binding [*query* query]
(let [[query-type druid-query] (build-druid-query query)]
(log-druid-query druid-query)
(->> (do-query druid-query)
(post-process query-type)
remove-bonus-keys))))
;;; ### MBQL Processor
(defn mbql->native
"Transpile an MBQL (inner) query into a native form suitable for a Druid DB."
[query]
;; Merge `:settings` into the inner query dict so the QP has access to it
(let [mbql-query (assoc (:query query)
:settings (:settings query))]
(binding [*query* mbql-query]
(let [[query-type druid-query] (build-druid-query mbql-query)]
{:query druid-query
:query-type query-type}))))
(defn execute-query
"Execute a query for a Druid DB."
[do-query {database :database, {:keys [query query-type]} :native}]
{:pre [database query]}
(let [details (:details database)
query (if (string? query)
(json/parse-string query keyword)
query)
query-type (or query-type (keyword "metabase.driver.druid.query-processor" (name (:queryType query))))
results (->> (do-query details query)
(post-process query-type)
remove-bonus-keys)
columns (vec (keys (first results)))]
{:columns columns
:rows (for [row results]
(mapv row columns))
:annotate? true}))
src/metabase/driver/generic_sql.clj
+ 3
− 4
View file @ fcf6ae33
......@@ -336,19 +336,18 @@
(defn IDriverSQLDefaultsMixin
"Default implementations of methods in `IDriver` for SQL drivers."
[]
(require 'metabase.driver.generic-sql.native
'metabase.driver.generic-sql.query-processor)
(require 'metabase.driver.generic-sql.query-processor)
(merge driver/IDriverDefaultsMixin
{:analyze-table analyze-table
:can-connect? can-connect?
:describe-database describe-database
:describe-table describe-table
:describe-table-fks describe-table-fks
:execute-query (resolve 'metabase.driver.generic-sql.query-processor/execute-query)
:features features
:field-values-lazy-seq field-values-lazy-seq
:mbql->native (resolve 'metabase.driver.generic-sql.query-processor/mbql->native)
:notify-database-updated notify-database-updated
:process-native (resolve 'metabase.driver.generic-sql.native/process-and-run)
:process-mbql (resolve 'metabase.driver.generic-sql.query-processor/process-mbql)
:table-rows-seq table-rows-seq}))
......
src/metabase/driver/generic_sql/native.clj deleted 100644 → 0
+ 0
− 50
View file @ 57ef8a8e
(ns metabase.driver.generic-sql.native
"The `native` query processor."
(:require [clojure.java.jdbc :as jdbc]
[clojure.tools.logging :as log]
[metabase.db :refer [sel]]
[metabase.driver :as driver]
[metabase.driver.generic-sql :as sql]
[metabase.models.database :refer [Database]]
[metabase.util :as u]))
(defn value->base-type
"Attempt to match a value we get back from the DB with the corresponding base-type`."
[v]
(driver/class->base-type (type v)))
(defn process-and-run
"Process and run a native (raw SQL) QUERY."
[driver {{sql :query} :native, database-id :database, settings :settings}]
(try (let [database (sel :one :fields [Database :id :engine :details] :id database-id)
db-conn (sql/db->jdbc-connection-spec database)]
(jdbc/with-db-transaction [t-conn db-conn]
(let [^java.sql.Connection jdbc-connection (:connection t-conn)]
;; Disable auto-commit for this transaction, that way shady queries are unable to modify the database
(.setAutoCommit jdbc-connection false)
(try
;; Set the timezone if applicable
(when-let [timezone (:report-timezone settings)]
(log/debug (u/format-color 'green "%s" (sql/set-timezone-sql driver)))
(try (jdbc/db-do-prepared t-conn (sql/set-timezone-sql driver) [timezone])
(catch Throwable e
(log/error (u/format-color 'red "Failed to set timezone: %s" (.getMessage e))))))
;; Now run the query itself
(log/debug (u/format-color 'green "%s" sql))
(let [[columns & [first-row :as rows]] (jdbc/query t-conn sql, :as-arrays? true)]
{:rows rows
:columns columns
:cols (for [[column first-value] (partition 2 (interleave columns first-row))]
{:name column
:base_type (value->base-type first-value)})})
;; Rollback any changes made during this transaction just to be extra-double-sure JDBC doesn't try to commit them automatically for us
(finally (.rollback jdbc-connection))))))
(catch java.sql.SQLException e
(let [^String message (or (->> (.getMessage e) ; error message comes back like 'Column "ZID" not found; SQL statement: ... [error-code]' sometimes
(re-find #"^(.*);") ; the user already knows the SQL, and error code is meaningless
second) ; so just return the part of the exception that is relevant
(.getMessage e))]
(throw (Exception. message))))))
src/metabase/driver/generic_sql/query_processor.clj
+ 45
− 36
View file @ fcf6ae33
......@@ -4,6 +4,9 @@
(clojure [string :as s]
[walk :as walk])
[clojure.tools.logging :as log]
[clj-time.coerce :as tc]
[clj-time.core :as t]
[clj-time.format :as tf]
(korma [core :as k]
[db :as kdb])
(korma.sql [engine :as kengine]
......@@ -288,45 +291,51 @@
(recur korma-form more)
korma-form))))
(defn- do-with-timezone [driver timezone f]
(log/debug (u/format-color 'blue (sql/set-timezone-sql driver)))
(try (kdb/transaction (k/exec-raw [(sql/set-timezone-sql driver) [timezone]])
(f))
(catch Throwable e
(log/error (u/format-color 'red "Failed to set timezone:\n%s"
(with-out-str (jdbc/print-sql-exception-chain e))))
(f))))
(defn- exception->nice-error-message ^String [^java.sql.SQLException e]
(or (->> (.getMessage e) ; error message comes back like "Error message ... [status-code]" sometimes
(re-find #"(?s)(^.*)\s+\[[\d-]+\]$") ; status code isn't useful and makes unit tests hard to write so strip it off
second) ; (?s) = Pattern.DOTALL - tell regex `.` to match newline characters as well
(.getMessage e)))
(defn- do-with-try-catch [f]
(try
(f)
(catch java.sql.SQLException e
(jdbc/print-sql-exception-chain e)
(throw (Exception. (exception->nice-error-message e))))))
(defn build-korma-form
"Build the korma form we will call `k/exec` on."
[driver {inner-query :query :as outer-query} entity]
(binding [*query* outer-query]
(apply-clauses driver (k/select* entity) inner-query)))
(defn process-mbql
"Convert QUERY into a korma `select` form, execute it, and annotate the results."
[driver {{:keys [source-table]} :query, database :database, settings :settings, :as outer-query}]
(let [timezone (:report-timezone settings)
entity ((resolve 'metabase.driver.generic-sql/korma-entity) database source-table)
korma-form (build-korma-form driver outer-query entity)
f (partial k/exec korma-form)
f (fn []
(kdb/with-db (:db entity)
(if (seq timezone)
(do-with-timezone driver timezone f)
(f))))]
(log-korma-form korma-form)
(do-with-try-catch f)))
(defn mbql->native
"Transpile MBQL query into a native SQL statement."
[driver {{:keys [source-table]} :query, database :database, :as outer-query}]
(let [entity ((resolve 'metabase.driver.generic-sql/korma-entity) database source-table)
korma-form (build-korma-form driver outer-query entity)
form-with-sql (kengine/bind-query korma-form (kengine/->sql korma-form))]
{:query (:sql-str form-with-sql)
:params (:params form-with-sql)}))
(defn execute-query
"Process and run a native (raw SQL) QUERY."
[driver {:keys [database settings], {sql :query, params :params} :native}]
(try (let [db-conn (sql/db->jdbc-connection-spec database)]
(jdbc/with-db-transaction [t-conn db-conn]
(let [^java.sql.Connection jdbc-connection (:connection t-conn)]
;; Disable auto-commit for this transaction, that way shady queries are unable to modify the database
(.setAutoCommit jdbc-connection false)
(try
;; Set the timezone if applicable
(when-let [timezone (:report-timezone settings)]
(log/debug (u/format-color 'green "%s" (sql/set-timezone-sql driver)))
(try (jdbc/db-do-prepared t-conn (sql/set-timezone-sql driver) [timezone])
(catch Throwable e
(log/error (u/format-color 'red "Failed to set timezone: %s" (.getMessage e))))))
;; Now run the query itself
(let [statement (if params
(into [sql] params)
sql)]
(let [[columns & rows] (jdbc/query t-conn statement, :identifiers identity, :as-arrays? true)]
{:rows rows
:columns columns}))
;; Rollback any changes made during this transaction just to be extra-double-sure JDBC doesn't try to commit them automatically for us
(finally (.rollback jdbc-connection))))))
(catch java.sql.SQLException e
(let [^String message (or (->> (.getMessage e) ; error message comes back like 'Column "ZID" not found; SQL statement: ... [error-code]' sometimes
(re-find #"^(.*);") ; the user already knows the SQL, and error code is meaningless
second) ; so just return the part of the exception that is relevant
(.getMessage e))]
(throw (Exception. message))))))
src/metabase/driver/h2.clj
+ 1
− 2
View file @ fcf6ae33
......@@ -6,7 +6,6 @@
[metabase.db :as db]
[metabase.driver :as driver]
[metabase.driver.generic-sql :as sql]
[metabase.models.database :refer [Database]]
[metabase.util :as u]
[metabase.util.korma-extensions :as kx]))
......@@ -122,7 +121,7 @@
;; For :native queries check to make sure the DB in question has a (non-default) NAME property specified in the connection string.
;; We don't allow SQL execution on H2 databases for the default admin account for security reasons
(when (= (keyword query-type) :native)
(let [{:keys [db]} (db/sel :one :field [Database :details] :id (:database query))
(let [{:keys [db]} (get-in query [:database :details])
_ (assert db)
[_ options] (connection-string->file+options db)
{:strs [USER]} options]
......
src/metabase/driver/mongo.clj
+ 6
− 11
View file @ fcf6ae33
......@@ -11,8 +11,7 @@
[metabase.driver :as driver]
(metabase.driver.mongo [query-processor :as qp]
[util :refer [*mongo-connection* with-mongo-connection values->base-type]])
(metabase.models [database :refer [Database]]
[field :as field]
(metabase.models [field :as field]
[table :as table])
[metabase.sync-database.analyze :as analyze]
[metabase.util :as u])
......@@ -42,13 +41,9 @@
message))
(defn- process-query-in-context [qp]
(fn [query]
(let [{:keys [database], :as query} (update query :database (fn [database]
(if (integer? database)
(Database database)
database)))]
(with-mongo-connection [^DB conn, database]
(qp query)))))
(fn [{:keys [database], :as query}]
(with-mongo-connection [^DB conn, database]
(qp query))))
;;; ### Syncing
......@@ -196,11 +191,11 @@
:display-name "Use a secure connection (SSL)?"
:type :boolean
:default false}])
:execute-query (u/drop-first-arg qp/execute-query)
:features (constantly #{:dynamic-schema :nested-fields})
:field-values-lazy-seq (u/drop-first-arg field-values-lazy-seq)
:humanize-connection-error-message (u/drop-first-arg humanize-connection-error-message)
:process-native (u/drop-first-arg qp/process-and-run-native)
:process-mbql (u/drop-first-arg qp/process-and-run-mbql)
:mbql->native (u/drop-first-arg qp/mbql->native)
:process-query-in-context (u/drop-first-arg process-query-in-context)
:sync-in-context (u/drop-first-arg sync-in-context)}))
......
src/metabase/driver/mongo/query_processor.clj
+ 30
− 27
View file @ fcf6ae33
......@@ -7,7 +7,6 @@
[cheshire.core :as json]
(monger [collection :as mc]
[operators :refer :all])
[metabase.db :as db]
[metabase.driver.mongo.util :refer [with-mongo-connection *mongo-connection* values->base-type]]
[metabase.models.table :refer [Table]]
[metabase.query-processor :as qp]
......@@ -40,27 +39,6 @@
(walk/postwalk #(if (symbol? %) (symbol (name %)) %)) ; strip namespace qualifiers from Monger form
u/pprint-to-str) "\n"))))
;; # NATIVE QUERY PROCESSOR
(defn process-and-run-native
"Process and run a native MongoDB query."
[{{query :query} :native, database :database, table-id :table, :as outer-query}]
{:pre [query (map? database) (integer? table-id)]}
(let [query (if (string? query)
(json/parse-string query keyword)
query)
results (mc/aggregate *mongo-connection* (db/sel :one :field [Table :name], :id table-id) query
:allow-disk-use true)]
;; As with Druid, we want to use `annotate` on the results of native queries.
;; Because the Generic SQL driver was written in ancient times, it has its own internal implementation of `annotate`
;; (which preserves the order of columns in the results) and as a result the `annotate` middleware isn't applied for `:native` queries.
;; Thus we need to manually run it ourself here.
;; TODO - it would be nice if we could let the middleware take care of `annotate` for us. This will likely require a new `IDriver` method
;; returns whether or not `annotate` should be done to native query results.
(annotate/annotate outer-query (if (sequential? results)
results
[results]))))
;;; # STRUCTURED QUERY PROCESSOR
......@@ -397,7 +375,8 @@
(u/->Timestamp (:___date v))
v)}))))
(defn process-and-run-mbql
(defn mbql->native
"Process and run an MBQL query."
[{database :database, {{source-table-name :name} :source-table} :query, :as query}]
{:pre [(map? database)
......@@ -405,7 +384,31 @@
(binding [*query* query]
(let [generated-pipeline (generate-aggregation-pipeline (:query query))]
(log-monger-form generated-pipeline)
(->> (mc/aggregate *mongo-connection* source-table-name generated-pipeline
:allow-disk-use true)
unescape-names
unstringify-dates))))
{:query generated-pipeline
:collection source-table-name
:mbql? true})))
(defn execute-query
"Process and run a native MongoDB query."
[{{:keys [collection query mbql?]} :native, database :database}]
{:pre [query
(string? collection)
(map? database)]}
(let [query (if (string? query)
(json/parse-string query keyword)
query)
results (mc/aggregate *mongo-connection* collection query
:allow-disk-use true)
results (if (sequential? results)
results
[results])
;; if we formed the query using MBQL then we apply a couple post processing functions
results (if-not mbql? results
(-> results
unescape-names
unstringify-dates))
columns (vec (keys (first results)))]
{:columns columns
:rows (for [row results]
(mapv row columns))
:annotate? true}))
src/metabase/query_processor.clj
+ 37
− 25
View file @ fcf6ae33
......@@ -9,6 +9,7 @@
(metabase [config :as config]
[db :as db]
[driver :as driver])
[metabase.models.database :as database]
(metabase.models [field :refer [Field]]
[query-execution :refer [QueryExecution]])
(metabase.query-processor [annotate :as annotate]
......@@ -144,13 +145,13 @@
"Transforms an MBQL into an expanded form with more information and structure. Also resolves references to fields, tables,
etc, into their concrete details which are necessary for query formation by the executing driver."
[qp]
(fn [query]
;; if necessary, expand/resolve the query
(let [query (if-not (mbql-query? query)
query
;; for MBQL queries we expand first, then resolve
(resolve/resolve (expand/expand query)))]
(qp query))))
(fn [{database-id :database, :as query}]
(let [resolved-db (db/sel :one :fields [database/Database :name :id :engine :details] :id database-id)
query (if-not (mbql-query? query)
query
;; for MBQL queries we expand first, then resolve
(resolve/resolve (expand/expand query)))]
(qp (assoc query :database resolved-db)))))
(defn- post-add-row-count-and-status
......@@ -375,18 +376,6 @@
absolute-max-results))))))
(defn post-annotate
"QP middleware that runs directly after the the query is run and adds metadata as appropriate."
[qp]
(fn [query]
(let [results (qp query)]
(if-not (mbql-query? query)
;; non-MBQL queries are not affected
results
;; for MBQL queries capture the results and annotate
(annotate/annotate query results)))))
(defn- pre-log-query [qp]
(fn [query]
(when (and (mbql-query? query)
......@@ -441,6 +430,33 @@
(assert (every? u/string-or-keyword? (:columns <>))))))))
(defn- run-query
"The end of the QP middleware which actually executes the query on the driver.
If this is an MBQL query then we first call `mbql->native` which builds a database dependent form for execution and
then we pass that form into the `execute-query` function for final execution.
If the query is already a *native* query then we simply pass it through to `execute-query` unmodified."
[query]
(let [native-form (u/prog1 (if-not (mbql-query? query)
(:native query)
(driver/mbql->native (:driver query) query))
(when-not *disable-qp-logging*
(log/debug (u/format-color 'green "NATIVE FORM:\n%s\n" (u/pprint-to-str <>)))))
native-query (if-not (mbql-query? query)
query
(assoc query :native native-form))
raw-result (driver/execute-query (:driver query) native-query)
query-result (if-not (or (mbql-query? query)
(:annotate? raw-result))
(assoc raw-result :columns (mapv name (:columns raw-result))
:cols (for [[column first-value] (partition 2 (interleave (:columns raw-result) (first (:rows raw-result))))]
{:name (name column)
:base_type (driver/class->base-type (type first-value))}))
(annotate/annotate query raw-result))]
(assoc query-result :native_form native-form)))
;;; +-------------------------------------------------------------------------------------------------------+
;;; | QUERY PROCESSOR |
;;; +-------------------------------------------------------------------------------------------------------+
......@@ -480,10 +496,7 @@
;; TODO: it probably makes sense to throw an error or return a failure response here if we can't get a driver
(let [driver (driver/database-id->driver (:database query))]
(binding [*driver* driver]
(let [driver-process-in-context (partial driver/process-query-in-context driver)
driver-process-query (partial (if (mbql-query? query)
driver/process-mbql
driver/process-native) driver)]
(let [driver-process-in-context (partial driver/process-query-in-context driver)]
((<<- wrap-catch-exceptions
pre-add-settings
pre-expand-macros
......@@ -497,10 +510,9 @@
cumulative-count
limit
post-check-results-format
post-annotate
pre-log-query
guard-multiple-calls
driver-process-query) (assoc query :driver driver))))))
run-query) (assoc query :driver driver))))))
;;; +----------------------------------------------------------------------------------------------------+
......
src/metabase/query_processor/annotate.clj
+ 6
− 5
View file @ fcf6ae33
......@@ -259,12 +259,13 @@
1. Sorts the results according to the rules at the top of this page
2. Resolves the Fields returned in the results and adds information like `:columns` and `:cols`
expected by the frontend."
[query results]
(let [result-keys (set (keys (first results)))
cols (resolve-sort-and-format-columns (:query query) result-keys)
columns (mapv :name cols)]
[query {:keys [columns rows]}]
(let [row-maps (for [row rows]
(zipmap columns row))
cols (resolve-sort-and-format-columns (:query query) (set columns))
columns (mapv :name cols)]
{:cols (vec (for [col cols]
(update col :name name)))
:columns (mapv name columns)
:rows (for [row results]
:rows (for [row row-maps]
(mapv row columns))}))
src/metabase/query_processor/resolve.clj
+ 1
− 8
View file @ fcf6ae33
......@@ -6,8 +6,7 @@
[medley.core :as m]
[schema.core :as s]
[metabase.db :refer [sel]]
(metabase.models [database :refer [Database]]
[field :as field]
(metabase.models [field :as field]
[table :refer [Table]])
[metabase.query-processor.interface :refer :all]
[metabase.util :as u])
......@@ -198,11 +197,6 @@
;; Recurse in case any new (nested) unresolved fields were found.
(recur (dec max-iterations))))))))
(defn- resolve-database
"Resolve the `Database` in question for an EXPANDED-QUERY-DICT."
[{database-id :database, :as expanded-query-dict}]
(assoc expanded-query-dict :database (sel :one :fields [Database :name :id :engine :details] :id database-id)))
(defn- join-tables-fetch-field-info
"Fetch info for PK/FK `Fields` for the JOIN-TABLES referenced in a Query."
[source-table-id join-tables fk-field-ids]
......@@ -255,5 +249,4 @@
(some-> expanded-query-dict
record-fk-field-ids
resolve-fields
resolve-database
resolve-tables))
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment