From 390896537c88509b87e3fd44c4fa3e51bd3d2596 Mon Sep 17 00:00:00 2001
From: Tom Robinson <tlrobinson@gmail.com>
Date: Tue, 13 Dec 2016 11:14:11 -0800
Subject: [PATCH] Tokenized editor
---
frontend/src/metabase/lib/dom.js | 40 +++
.../src/metabase/lib/expressions/parser.js | 286 +++++++++++++-----
.../expressions/ExpressionEditorTextfield.jsx | 45 +--
.../expressions/TokenizedExpression.css | 56 ++++
.../expressions/TokenizedExpression.jsx | 54 ++++
.../components/expressions/TokenizedInput.jsx | 122 ++++++++
.../unit/lib/expressions/formatter.spec.js | 2 +-
.../test/unit/lib/expressions/parser.spec.js | 128 +++++---
frontend/test/unit/lib/query.spec.js | 2 +-
package.json | 2 +-
yarn.lock | 6 +-
11 files changed, 602 insertions(+), 141 deletions(-)
create mode 100644 frontend/src/metabase/query_builder/components/expressions/TokenizedExpression.css
create mode 100644 frontend/src/metabase/query_builder/components/expressions/TokenizedExpression.jsx
create mode 100644 frontend/src/metabase/query_builder/components/expressions/TokenizedInput.jsx
diff --git a/frontend/src/metabase/lib/dom.js b/frontend/src/metabase/lib/dom.js
index 014ccbc784c..a5c93b10684 100644
--- a/frontend/src/metabase/lib/dom.js
+++ b/frontend/src/metabase/lib/dom.js
@@ -55,6 +55,18 @@ export function elementIsInView(element, percentX = 1, percentY = 1) {
});
}
+export function getCaretPosition(element) {
+ if (element.nodeName.toLowerCase() === "input" || element.nodeName.toLowerCase() === "textarea") {
+ return element.selectionStart;
+ } else {
+ // contenteditable
+ const selection = window.getSelection();
+ const range = selection.getRangeAt(0);
+ range.setStart(element, 0);
+ return range.toString().length;
+ }
+}
+
export function setCaretPosition(element, position) {
if (element.setSelectionRange) {
element.focus();
@@ -65,5 +77,33 @@ export function setCaretPosition(element, position) {
range.moveEnd("character", position);
range.moveStart("character", position);
range.select();
+ } else {
+ // contenteditable
+ const selection = window.getSelection();
+ const pos = getTextNodeAtPosition(element, position);
+ selection.removeAllRanges();
+ const range = new Range();
+ range.setStart(pos.node ,pos.position);
+ selection.addRange(range);
}
}
+
+export function saveCaretPosition(context) {
+ let position = getCaretPosition(context);
+ return () => setCaretPosition(context, position);
+}
+
+function getTextNodeAtPosition(root, index) {
+ let treeWalker = document.createTreeWalker(root, NodeFilter.SHOW_TEXT, (elem) => {
+ if (index > elem.textContent.length){
+ index -= elem.textContent.length;
+ return NodeFilter.FILTER_REJECT
+ }
+ return NodeFilter.FILTER_ACCEPT;
+ });
+ var c = treeWalker.nextNode();
+ return {
+ node: c ? c : root,
+ position: c ? index : 0
+ };
+}
diff --git a/frontend/src/metabase/lib/expressions/parser.js b/frontend/src/metabase/lib/expressions/parser.js
index 7281b75c527..e84c922642a 100644
--- a/frontend/src/metabase/lib/expressions/parser.js
+++ b/frontend/src/metabase/lib/expressions/parser.js
@@ -43,13 +43,13 @@ const ExpressionsLexer = new Lexer(allTokens);
class ExpressionsParser extends Parser {
- constructor(input, options) {
- super(input, allTokens, { recoveryEnabled: false });
-
- this._options = options;
+ constructor(input, options = {}) {
+ super(input, allTokens/*, { recoveryEnabled: false }*/);
let $ = this;
+ this._options = options;
+
// an expression without aggregations in it
$.RULE("expression", function (outsideAggregation = false) {
return $.SUBRULE($.additionExpression, [outsideAggregation])
@@ -64,35 +64,23 @@ class ExpressionsParser extends Parser {
// The precedence of binary expressions is determined by
// how far down the Parse Tree the binary expression appears.
$.RULE("additionExpression", (outsideAggregation) => {
- let value = $.SUBRULE($.multiplicationExpression, [outsideAggregation]);
- $.MANY(() => {
+ let initial = $.SUBRULE($.multiplicationExpression, [outsideAggregation]);
+ let operations = $.MANY(() => {
const op = $.CONSUME(AdditiveOperator);
const rhsVal = $.SUBRULE2($.multiplicationExpression, [outsideAggregation]);
-
- // collapse multiple consecutive operators into a single MBQL statement
- if (Array.isArray(value) && value[0] === op.image) {
- value.push(rhsVal);
- } else {
- value = [op.image, value, rhsVal]
- }
+ return [op, rhsVal];
});
- return value;
+ return this._math(initial, operations);
});
$.RULE("multiplicationExpression", (outsideAggregation) => {
- let value = $.SUBRULE($.atomicExpression, [outsideAggregation]);
- $.MANY(() => {
+ let initial = $.SUBRULE($.atomicExpression, [outsideAggregation]);
+ let operations = $.MANY(() => {
const op = $.CONSUME(MultiplicativeOperator);
const rhsVal = $.SUBRULE2($.atomicExpression, [outsideAggregation]);
-
- // collapse multiple consecutive operators into a single MBQL statement
- if (Array.isArray(value) && value[0] === op.image) {
- value.push(rhsVal);
- } else {
- value = [op.image, value, rhsVal]
- }
+ return [op, rhsVal];
});
- return value;
+ return this._math(initial, operations);
});
$.RULE("aggregationOrMetricExpression", (outsideAggregation) => {
@@ -103,48 +91,58 @@ class ExpressionsParser extends Parser {
});
$.RULE("aggregationExpression", (outsideAggregation) => {
- const agg = $.CONSUME(Aggregation).image;
- let value = [aggregationsMap.get(agg)]
- $.CONSUME(LParen);
- $.OPTION(() => {
- // aggregations cannot be nested, so pass false to the expression subrule
- value.push($.SUBRULE($.expression, [false]));
- $.MANY(() => {
- $.CONSUME(Comma);
- value.push($.SUBRULE2($.expression, [false]));
- });
- });
- $.CONSUME(RParen);
- return value;
+ const aggregation = $.CONSUME(Aggregation);
+ const lParen = $.CONSUME(LParen);
+ const args = $.MANY_SEP(Comma, () => $.SUBRULE($.expression, [false]));
+ const rParen = $.CONSUME(RParen);
+
+ return this._aggregation(aggregation, lParen, args, rParen);
});
$.RULE("metricExpression", () => {
- let metricName = $.CONSUME(Identifier).image;
- $.CONSUME(LParen);
- $.CONSUME(RParen);
- const metric = this.getMetricForName(metricName);
+ const metricName = $.SUBRULE($.identifier);
+ const lParen = $.CONSUME(LParen);
+ const rParen = $.CONSUME(RParen);
+
+ const metric = this.getMetricForName(this._toString(metricName));
if (metric != null) {
- return ["METRIC", metric.id];
+ return this._metricReference(metricName, metric.id);
}
- throw new Error("Unknown metric \"" + metricName + "\"");
+ return this._unknownMetric(metricName, lParen, rParen);
});
$.RULE("fieldExpression", () => {
- let fieldName = $.OR([
- {ALT: () => JSON.parse($.CONSUME(StringLiteral).image) },
- {ALT: () => $.CONSUME(Identifier).image }
+ const fieldName = $.OR([
+ {ALT: () => $.SUBRULE($.stringLiteral) },
+ {ALT: () => $.SUBRULE($.identifier) }
]);
- const field = this.getFieldForName(fieldName);
+
+ const field = this.getFieldForName(this._toString(fieldName));
if (field != null) {
- return ["field-id", field.id];
+ return this._fieldReference(fieldName, field.id);
}
- const expression = this.getExpressionForName(fieldName);
+ const expression = this.getExpressionForName(this._toString(fieldName));
if (expression != null) {
- return ["expression", fieldName];
+ return this._expressionReference(fieldName, expression);
}
- throw new Error("Unknown field \"" + fieldName + "\"");
+ return this._unknownField(fieldName);
});
+ $.RULE("identifier", () => {
+ const identifier = $.CONSUME(Identifier);
+ return this._identifier(identifier);
+ })
+
+ $.RULE("stringLiteral", () => {
+ const stringLiteral = $.CONSUME(StringLiteral);
+ return this._stringLiteral(stringLiteral);
+ })
+
+ $.RULE("numberLiteral", () => {
+ const numberLiteral = $.CONSUME(NumberLiteral);
+ return this._numberLiteral(numberLiteral);
+ })
+
$.RULE("atomicExpression", (outsideAggregation) => {
return $.OR([
// aggregations not allowed inside other aggregations
@@ -152,18 +150,15 @@ class ExpressionsParser extends Parser {
// fields not allowed outside aggregations
{GATE: () => !outsideAggregation, ALT: () => $.SUBRULE($.fieldExpression) },
{ALT: () => $.SUBRULE($.parenthesisExpression, [outsideAggregation]) },
- {ALT: () => parseFloat($.CONSUME(NumberLiteral).image) }
+ {ALT: () => $.SUBRULE($.numberLiteral) }
], "a number or field name");
});
$.RULE("parenthesisExpression", (outsideAggregation) => {
- let expValue;
-
- $.CONSUME(LParen);
- expValue = $.SUBRULE($.expression, [outsideAggregation]);
- $.CONSUME(RParen);
-
- return expValue
+ let lParen = $.CONSUME(LParen);
+ let expValue = $.SUBRULE($.expression, [outsideAggregation]);
+ let rParen = $.CONSUME(RParen);
+ return this._parens(lParen, expValue, rParen);
});
Parser.performSelfAnalysis(this);
@@ -185,6 +180,119 @@ class ExpressionsParser extends Parser {
}
}
+class ExpressionsParserMBQL extends ExpressionsParser {
+ _math(initial, operations) {
+ for (const [op, rhsVal] of operations) {
+ // collapse multiple consecutive operators into a single MBQL statement
+ if (Array.isArray(initial) && initial[0] === op.image) {
+ initial.push(rhsVal);
+ } else {
+ initial = [op.image, initial, rhsVal]
+ }
+ }
+ return initial;
+ }
+ _aggregation(aggregation, lParen, args, rParen) {
+ const aggregationName = aggregation.image;
+ return [aggregationsMap.get(aggregationName)].concat(args.values);
+ }
+ _metricReference(metricName, metricId) {
+ return ["METRIC", metricId];
+ }
+ _fieldReference(fieldName, fieldId) {
+ return ["field-id", fieldId];
+ }
+ _expressionReference(fieldName) {
+ return ["expression", fieldName];
+ }
+ _unknownField(fieldName) {
+ throw new Error("Unknown field \"" + fieldName + "\"");
+ }
+ _unknownMetric(metricName) {
+ throw new Error("Unknown metric \"" + metricName + "\"");
+ }
+
+ _identifier(identifier) {
+ return identifier.image;
+ }
+ _stringLiteral(stringLiteral) {
+ return JSON.parse(stringLiteral.image);
+ }
+ _numberLiteral(numberLiteral) {
+ return parseFloat(numberLiteral.image);
+ }
+ _parens(lParen, expValue, rParen) {
+ return expValue;
+ }
+ _toString(x) {
+ return x;
+ }
+}
+
+const syntax = (type, ...children) => ({
+ type: type,
+ children: children
+})
+const token = (token) => ({
+ type: "token",
+ text: token.image,
+ start: token.startOffset,
+ end: token.endOffset,
+})
+
+class ExpressionsParserSyntax extends ExpressionsParser {
+ _math(initial, operations) {
+ return syntax("math", ...[initial].concat(...operations.map(([op, arg]) => [token(op), arg])));
+ }
+ _aggregation(aggregation, lParen, args, rParen) {
+ let argsAndCommas = [];
+ for (var i = 0; i < args.values.length; i++) {
+ argsAndCommas.push(args.values[i]);
+ if (i < args.separators.length) {
+ argsAndCommas.push(args.separators[i]);
+ }
+ }
+ return syntax("aggregation", token(aggregation), token(lParen), ...argsAndCommas, token(rParen));
+ }
+ _metricReference(metricName, metricId) {
+ return syntax("metric", metricName);
+ }
+ _fieldReference(fieldName, fieldId) {
+ return syntax("field", fieldName);
+ }
+ _expressionReference(fieldName) {
+ return syntax("expression-reference", token(fieldName));
+ }
+ _unknownField(fieldName) {
+ return syntax("unknown", fieldName);
+ }
+ _unknownMetric(metricName) {
+ return syntax("unknown", metricName);
+ }
+
+ _identifier(identifier) {
+ return syntax("identifier", token(identifier));
+ }
+ _stringLiteral(stringLiteral) {
+ return syntax("string", token(stringLiteral));
+ }
+ _numberLiteral(numberLiteral) {
+ return syntax("number", token(numberLiteral));
+ }
+ _parens(lParen, expValue, rParen) {
+ return syntax("group", token(lParen), expValue, token(rParen));
+ }
+ _toString(x) {
+ if (typeof x === "string") {
+ return x;
+ } else if (x.type === "string") {
+ return JSON.parse(x.children[0].text);
+ } else if (x.type === "identifier") {
+ return x.children[0].text;
+ }
+ }
+}
+
function getSubTokenTypes(TokenClass) {
return TokenClass.extendingTokenTypes.map(tokenType => _.findWhere(allTokens, { tokenType }));
}
@@ -194,12 +302,12 @@ function getTokenSource(TokenClass) {
return TokenClass.PATTERN.source.replace(/^\\/, "");
}
-export function compile(source, options = {}) {
+function run(Parser, source, options) {
if (!source) {
return [];
}
const { startRule } = options;
- const parser = new ExpressionsParser(ExpressionsLexer.tokenize(source).tokens, options);
+ const parser = new Parser(ExpressionsLexer.tokenize(source).tokens, options);
const expression = parser[startRule]();
if (parser.errors.length > 0) {
throw parser.errors;
@@ -207,6 +315,14 @@ export function compile(source, options = {}) {
return expression;
}
+export function compile(source, options = {}) {
+ return run(ExpressionsParserMBQL, source, options);
+}
+
+export function parse(source, options = {}) {
+ return run(ExpressionsParserSyntax, source, options);
+}
+
// No need for more than one instance.
const parserInstance = new ExpressionsParser([])
export function suggest(source, {
@@ -232,13 +348,13 @@ export function suggest(source, {
partialSuggestionMode = true
}
- const syntacticSuggestions = parserInstance.computeContentAssist(startRule, assistanceTokenVector)
let finalSuggestions = []
// TODO: is there a better way to figure out which aggregation we're inside of?
const currentAggregationToken = _.find(assistanceTokenVector.slice().reverse(), (t) => t instanceof Aggregation);
+ const syntacticSuggestions = parserInstance.computeContentAssist(startRule, assistanceTokenVector)
for (const suggestion of syntacticSuggestions) {
const { nextTokenType, ruleStack } = suggestion;
// no nesting of aggregations or field references outside of aggregations
@@ -273,25 +389,29 @@ export function suggest(source, {
postfixTrim: /^\s*\)?\s*/
});
} else if (nextTokenType === Identifier || nextTokenType === StringLiteral) {
- if (!outsideAggregation && currentAggregationToken) {
- let aggregationShort = aggregationsMap.get(getImage(currentAggregationToken));
- let aggregationOption = _.findWhere(tableMetadata.aggregation_options, { short: aggregationShort });
- if (aggregationOption && aggregationOption.fields.length > 0) {
- finalSuggestions.push(...aggregationOption.fields[0].map(field => ({
- type: "fields",
- name: field.display_name,
- text: formatFieldName(field) + " ",
- prefixTrim: /\w+$/,
- postfixTrim: /^\w+\s*/
- })))
- finalSuggestions.push(...Object.keys(customFields || {}).map(expressionName => ({
- type: "fields",
- name: expressionName,
- text: formatExpressionName(expressionName) + " ",
- prefixTrim: /\w+$/,
- postfixTrim: /^\w+\s*/
- })))
+ if (!outsideAggregation) {
+ let fields = [];
+ if (startRule === "aggregation" && currentAggregationToken) {
+ let aggregationShort = aggregationsMap.get(getImage(currentAggregationToken));
+ let aggregationOption = _.findWhere(tableMetadata.aggregation_options, { short: aggregationShort });
+ fields = aggregationOption && aggregationOption.fields && aggregationOption.fields[0] || []
+ } else if (startRule === "expression") {
+ fields = tableMetadata.fields;
}
+ finalSuggestions.push(...fields.map(field => ({
+ type: "fields",
+ name: field.display_name,
+ text: formatFieldName(field) + " ",
+ prefixTrim: /\w+$/,
+ postfixTrim: /^\w+\s*/
+ })));
+ finalSuggestions.push(...Object.keys(customFields || {}).map(expressionName => ({
+ type: "fields",
+ name: expressionName,
+ text: formatExpressionName(expressionName) + " ",
+ prefixTrim: /\w+$/,
+ postfixTrim: /^\w+\s*/
+ })));
}
} else if (nextTokenType === Aggregation) {
if (outsideAggregation) {
@@ -303,15 +423,15 @@ export function suggest(source, {
text: formatAggregationName(aggregationOption) + "(" + (arity > 0 ? "" : ")"),
postfixText: arity > 0 ? ")" : "",
prefixTrim: /\w+$/,
- postfixTrim: /^\w+\(\)?/
+ postfixTrim: /^\w+(\(\)?|$)/
};
}));
finalSuggestions.push(...tableMetadata.metrics.map(metric => ({
type: "metrics",
name: metric.name,
- text: formatMetricName(metric) + "() ",
+ text: formatMetricName(metric) + "()",
prefixTrim: /\w+$/,
- postfixTrim: /^\w+\(\)?/
+ postfixTrim: /^\w+(\(\)?|$)/
})))
}
} else if (nextTokenType === NumberLiteral) {
diff --git a/frontend/src/metabase/query_builder/components/expressions/ExpressionEditorTextfield.jsx b/frontend/src/metabase/query_builder/components/expressions/ExpressionEditorTextfield.jsx
index 72fc3ccf5e0..f839d50d360 100644
--- a/frontend/src/metabase/query_builder/components/expressions/ExpressionEditorTextfield.jsx
+++ b/frontend/src/metabase/query_builder/components/expressions/ExpressionEditorTextfield.jsx
@@ -7,10 +7,12 @@ import cx from "classnames";
import { compile, suggest } from "metabase/lib/expressions/parser";
import { format } from "metabase/lib/expressions/formatter";
-import { setCaretPosition } from "metabase/lib/dom";
+import { setCaretPosition, getCaretPosition } from "metabase/lib/dom";
import Popover from "metabase/components/Popover.jsx";
+import TokenizedInput from "./TokenizedInput.jsx";
+
import { isExpression } from "metabase/lib/expressions";
@@ -26,7 +28,7 @@ const KEYCODE_DOWN = 40;
export default class ExpressionEditorTextfield extends Component {
constructor(props, context) {
super(props, context);
- _.bindAll(this, 'onInputChange', 'onInputKeyDown', 'onInputBlur', 'onSuggestionAccepted', 'onSuggestionMouseDown');
+ _.bindAll(this, '_triggerAutosuggest', 'onInputKeyDown', 'onInputBlur', 'onSuggestionAccepted', 'onSuggestionMouseDown');
}
static propTypes = {
@@ -81,12 +83,10 @@ export default class ExpressionEditorTextfield extends Component {
}
componentDidMount() {
- // causes the autocomplete widget to open immediately
- this.onInputChange();
+ this._setCaretPosition(this.state.expressionString.length, this.state.expressionString.length === 0)
}
onSuggestionAccepted() {
- let inputElement = ReactDOM.findDOMNode(this.refs.input);
const { expressionString } = this.state;
const suggestion = this.state.suggestions[this.state.highlightedSuggestion];
@@ -99,11 +99,12 @@ export default class ExpressionEditorTextfield extends Component {
if (suggestion.postfixTrim) {
postfix = postfix.replace(suggestion.postfixTrim, "");
}
+ if (!postfix && suggestion.postfixText) {
+ postfix = suggestion.postfixText;
+ }
- inputElement.value = prefix + suggestion.text + postfix;
- inputElement.focus();
- setCaretPosition(inputElement, (prefix + suggestion.text).length);
- this.onInputChange();
+ this.onExpressionChange(prefix + suggestion.text + postfix)
+ setTimeout(() => this._setCaretPosition((prefix + suggestion.text).length, true))
}
this.setState({
@@ -122,7 +123,7 @@ export default class ExpressionEditorTextfield extends Component {
const { suggestions, highlightedSuggestion } = this.state;
if (event.keyCode === KEYCODE_LEFT || event.keyCode === KEYCODE_RIGHT) {
- setTimeout(() => this.onInputChange());
+ setTimeout(() => this._triggerAutosuggest());
return;
}
if (event.keyCode === KEYCODE_ESC) {
@@ -170,15 +171,25 @@ export default class ExpressionEditorTextfield extends Component {
}
onInputClick = () => {
- this.onInputChange();
+ this._triggerAutosuggest();
+ }
+
+ _triggerAutosuggest = () => {
+ this.onExpressionChange(this.state.expressionString);
+ }
+
+ _setCaretPosition = (position, autosuggest) => {
+ setCaretPosition(ReactDOM.findDOMNode(this.refs.input), position);
+ if (autosuggest) {
+ setTimeout(() => this._triggerAutosuggest());
+ }
}
- onInputChange() {
+ onExpressionChange(expressionString) {
let inputElement = ReactDOM.findDOMNode(this.refs.input);
if (!inputElement) {
return;
}
- let expressionString = inputElement.value;
let expressionErrorMessage = null;
let suggestions = [];
@@ -199,7 +210,7 @@ export default class ExpressionEditorTextfield extends Component {
tableMetadata: this.props.tableMetadata,
customFields: this.props.customFields,
startRule: this.props.startRule,
- index: inputElement.selectionStart
+ index: getCaretPosition(inputElement)
})
} catch (e) {
console.error("suggest error:", e);
@@ -222,16 +233,16 @@ export default class ExpressionEditorTextfield extends Component {
return (
<div className={cx(S.editor, "relative")}>
- <input
+ <TokenizedInput
ref="input"
className={cx(S.input, "my1 p1 input block full h4 text-dark", { "border-error": errorMessage })}
type="text"
placeholder={placeholder}
value={this.state.expressionString}
- onChange={this.onInputChange}
+ onChange={(e) => this.onExpressionChange(e.target.value)}
onKeyDown={this.onInputKeyDown}
onBlur={this.onInputBlur}
- onFocus={this.onInputChange}
+ onFocus={(e) => this._triggerAutosuggest()}
onClick={this.onInputClick}
autoFocus
/>
diff --git a/frontend/src/metabase/query_builder/components/expressions/TokenizedExpression.css b/frontend/src/metabase/query_builder/components/expressions/TokenizedExpression.css
new file mode 100644
index 00000000000..92e73bcbd32
--- /dev/null
+++ b/frontend/src/metabase/query_builder/components/expressions/TokenizedExpression.css
@@ -0,0 +1,56 @@
+
+/*
+.TokenizedExpression .aggregation { border: 3px solid rgb(255,0,0) !important; }
+.TokenizedExpression .group { border: 3px solid rgb(0,255,0) !important; }
+.TokenizedExpression .identifier { border: 3px solid rgb(0,0,255) !important; }
+.TokenizedExpression .string-literal { border: 3px solid rgb(0,255,255) !important; }
+.TokenizedExpression .open-paren { border: 3px solid rgb(255,255,0) !important; }
+.TokenizedExpression .close-paren { border: 3px solid rgb(0,255,255) !important; } /**/
+
+.TokenizedExpression .aggregation,
+.TokenizedExpression .group,
+.TokenizedExpression .identifier,
+.TokenizedExpression .open-paren,
+.TokenizedExpression .close-paren,
+.TokenizedExpression .string-literal {
+ display: inline-block;
+ border-radius: 4px;
+/* display: flex;
+ align-items: center; */
+}
+
+.TokenizedExpression .string-literal > .open-quote,
+.TokenizedExpression .string-literal > .close-quote {
+ /* don't use `display: none;` `visibility: hidden;` `width: 0;` or `height: 0;` as text won't appear when copied */
+ /* display: inline-block; */
+ opacity: 0.5;
+ /*width: 1px;
+ height: 1px; */
+ /* overflow: hidden; */
+}
+
+.TokenizedExpression .identifier {
+ border: 1px solid #84BB4C;
+ background-color: #bfe49b;
+ padding: 0px 2px;
+ margin: 1px 1px;
+}
+
+.TokenizedExpression .aggregation .identifier {
+ border: 1px solid #2D86D4;
+ background-color: #6ab9ff;
+}
+
+.TokenizedExpression .aggregation {
+ border: 1px solid #84BB4C;
+ background-color: #bfe49b;
+ padding: 0px 2px;
+ /* height: 25px; */
+}
+
+.TokenizedExpression .aggregation > .identifier {
+ border: none;
+ background-color: transparent;
+ padding: 0;
+ margin: 0;
+}
diff --git a/frontend/src/metabase/query_builder/components/expressions/TokenizedExpression.jsx b/frontend/src/metabase/query_builder/components/expressions/TokenizedExpression.jsx
new file mode 100644
index 00000000000..406a9576469
--- /dev/null
+++ b/frontend/src/metabase/query_builder/components/expressions/TokenizedExpression.jsx
@@ -0,0 +1,54 @@
+import React, { Component, PropTypes } from "react";
+
+import "./TokenizedExpression.css";
+
+function nextNonWhitespace(tokens, index) {
+ while (index < tokens.length && /^\s+$/.test(tokens[++index])) {
+ }
+ return tokens[index];
+}
+
+export default class TokenizedExpression extends React.Component {
+ render() {
+ let tokens = this.props.source.match(/[a-zA-Z]\w*|"(?:[^\\"]+|\\(?:[bfnrtv"\\/]|u[0-9a-fA-F]{4}))*"|\(|\)|\d+|\s+|[*/+-]|.*/g);
+ let root = <span className="TokenizedExpression" children={[]} />;
+ let current = root;
+ const stack = [];
+ const push = (element) => {
+ current.props.children.push(element);
+ stack.push(current);
+ current = element;
+ }
+ const pop = () => {
+ if (stack.length === 0) {
+ return;
+ }
+ current = stack.pop();
+ }
+ for (let i = 0; i < tokens.length; i++) {
+ let token = tokens[i];
+ if (/^[a-zA-Z]\w*$/.test(token)) {
+ if (nextNonWhitespace(tokens, i) === "(") {
+ push(<span className="aggregation" children={[]} />);
+ }
+ current.props.children.push(<span className="identifier">{token}</span>);
+ } else if (/^"(?:[^\\"]+|\\(?:[bfnrtv"\\/]|u[0-9a-fA-F]{4}))*"$/.test(token)) {
+ current.props.children.push(
+ <span className="string-literal"><span className="open-quote">"</span><span className="identifier">{JSON.parse(token)}</span><span className="close-quote">"</span></span>
+ );
+ } else if (token === "(") {
+ push(<span className="group" children={[]} />)
+ current.props.children.push(<span className="open-paren">(</span>)
+ } else if (token === ")") {
+ current.props.children.push(<span className="close-paren">)</span>)
+ pop();
+ if (current.props.className === "aggregation") {
+ pop();
+ }
+ } else {
+ current.props.children.push(token);
+ }
+ }
+ return root;
+ }
+}
diff --git a/frontend/src/metabase/query_builder/components/expressions/TokenizedInput.jsx b/frontend/src/metabase/query_builder/components/expressions/TokenizedInput.jsx
new file mode 100644
index 00000000000..fe1993633de
--- /dev/null
+++ b/frontend/src/metabase/query_builder/components/expressions/TokenizedInput.jsx
@@ -0,0 +1,122 @@
+import React, { Component, PropTypes } from "react";
+import ReactDOM from "react-dom";
+
+import TokenizedExpression from "./TokenizedExpression.jsx";
+
+import { getCaretPosition, saveCaretPosition } from "metabase/lib/dom"
+
+export default class TokenizedInput extends Component {
+ constructor(props) {
+ super(props);
+ this.state = {
+ value: ""
+ }
+ }
+
+ _getValue() {
+ if (this.props.value != undefined) {
+ return this.props.value;
+ } else {
+ return this.state.value;
+ }
+ }
+ _setValue(value) {
+ ReactDOM.findDOMNode(this).value = value;
+ if (typeof this.props.onChange === "function") {
+ this.props.onChange({ target: { value }});
+ } else {
+ this.setState({ value });
+ }
+ }
+
+ componentDidMount() {
+ ReactDOM.findDOMNode(this).focus();
+ this.componentDidUpdate()
+
+ document.addEventListener("selectionchange", this.onSelectionChange, false);
+ }
+ componentWillUnmount() {
+ document.removeEventListener("selectionchange", this.onSelectionChange, false);
+ }
+ onSelectionChange = (e) => {
+ ReactDOM.findDOMNode(this).selectionStart = getCaretPosition(ReactDOM.findDOMNode(this))
+ }
+ onInput = (e) => {
+ this._setValue(e.target.textContent);
+ }
+ onKeyDown = (e) => {
+ this.props.onKeyDown(e);
+
+ // handle tokenized delete
+ if (e.keyCode !== 8) {
+ return;
+ }
+ const input = ReactDOM.findDOMNode(this);
+
+ var selection = window.getSelection();
+ var range = selection.getRangeAt(0);
+
+ let isEndOfNode = range.endContainer.length === range.endOffset;
+ let hasSelection = range.startContainer !== range.endContainer || range.startOffset !== range.endOffset;
+
+ let el = selection.focusNode;
+ let path = [];
+ while (el && el != input) {
+ path.unshift(el.className);
+ el = el.parentNode;
+ }
+
+ /*
+ e.stopPropagation();
+ e.preventDefault();
+ return;
+ /**/
+
+ if (!isEndOfNode || hasSelection) {
+ return;
+ }
+
+ let parent = selection.focusNode.parentNode;
+ if (parent.className === "close-paren" && parent.parentNode.className === "group" && parent.parentNode.parentNode.className === "aggregation") {
+ parent.parentNode.parentNode.parentNode.removeChild(parent.parentNode.parentNode);
+ e.stopPropagation();
+ e.preventDefault();
+ this._setValue(input.textContent);
+ } else if (parent.className === "identifier") {
+ if (parent.parentNode.className === "string-literal") {
+ parent.parentNode.parentNode.removeChild(parent.parentNode);
+ } else {
+ parent.parentNode.removeChild(parent);
+ }
+ e.stopPropagation();
+ e.preventDefault();
+ this._setValue(input.textContent);
+ }
+ }
+
+ componentDidUpdate() {
+ const inputNode = ReactDOM.findDOMNode(this);
+ const restore = saveCaretPosition(inputNode);
+
+ ReactDOM.unmountComponentAtNode(inputNode);
+ while (inputNode.firstChild) {
+ inputNode.removeChild(inputNode.firstChild);
+ }
+ ReactDOM.render(<TokenizedExpression source={this._getValue()} />, inputNode);
+ restore();
+ }
+ render() {
+ const { className, onFocus, onBlur, onClick } = this.props;
+ return (
+ <div
+ className={className}
+ contentEditable
+ onKeyDown={this.onKeyDown}
+ onInput={this.onInput}
+ onFocus={onFocus}
+ onBlur={onBlur}
+ onClick={onClick}
+ />
+ );
+ }
+}
diff --git a/frontend/test/unit/lib/expressions/formatter.spec.js b/frontend/test/unit/lib/expressions/formatter.spec.js
index 00a65395f86..458cf75483c 100644
--- a/frontend/test/unit/lib/expressions/formatter.spec.js
+++ b/frontend/test/unit/lib/expressions/formatter.spec.js
@@ -14,7 +14,7 @@ const mockMetadata = {
}
}
-fdescribe("lib/expressions/parser", () => {
+describe("lib/expressions/parser", () => {
describe("format", () => {
it("can format simple expressions", () => {
expect(format(["+", ["field-id", 1], ["field-id", 2]], mockMetadata)).toEqual("A + B");
diff --git a/frontend/test/unit/lib/expressions/parser.spec.js b/frontend/test/unit/lib/expressions/parser.spec.js
index ca5ac4addc3..91138bae50b 100644
--- a/frontend/test/unit/lib/expressions/parser.spec.js
+++ b/frontend/test/unit/lib/expressions/parser.spec.js
@@ -1,12 +1,26 @@
-import { compile, suggest } from "metabase/lib/expressions/parser";
+import { compile, suggest, parse } from "metabase/lib/expressions/parser";
import _ from "underscore";
-const mockFields = [
- {id: 1, display_name: "A"},
- {id: 2, display_name: "B"},
- {id: 3, display_name: "C"},
- {id: 10, display_name: "Toucan Sam"}
-];
+const mockMetadata = {
+ tableMetadata: {
+ fields: [
+ {id: 1, display_name: "A"},
+ {id: 2, display_name: "B"},
+ {id: 3, display_name: "C"},
+ {id: 10, display_name: "Toucan Sam"}
+ ],
+ metrics: [
+ {id: 1, name: "foo bar"},
+ ],
+ aggregation_options: [
+ { short: "count", fields: [] },
+ { short: "sum", fields: [[]] }
+ ]
+ }
+}
+
+const expressionOpts = { ...mockMetadata, startRule: "expression" };
+const aggregationOpts = { ...mockMetadata, startRule: "aggregation" };
describe("lib/expressions/parser", () => {
describe("compile()", () => {
@@ -17,51 +31,55 @@ describe("lib/expressions/parser", () => {
});
it("can parse simple expressions", () => {
- expect(compile("A", { fields: mockFields })).toEqual(['field-id', 1]);
- expect(compile("1", { fields: mockFields })).toEqual(1);
- expect(compile("1.1", { fields: mockFields })).toEqual(1.1);
+ expect(compile("A", expressionOpts)).toEqual(['field-id', 1]);
+ expect(compile("1", expressionOpts)).toEqual(1);
+ expect(compile("1.1", expressionOpts)).toEqual(1.1);
});
it("can parse single operator math", () => {
- expect(compile("A-B", { fields: mockFields })).toEqual(["-", ['field-id', 1], ['field-id', 2]]);
- expect(compile("A - B", { fields: mockFields })).toEqual(["-", ['field-id', 1], ['field-id', 2]]);
- expect(compile("1 - B", { fields: mockFields })).toEqual(["-", 1, ['field-id', 2]]);
- expect(compile("1 - 2", { fields: mockFields })).toEqual(["-", 1, 2]);
+ expect(compile("A-B", expressionOpts)).toEqual(["-", ['field-id', 1], ['field-id', 2]]);
+ expect(compile("A - B", expressionOpts)).toEqual(["-", ['field-id', 1], ['field-id', 2]]);
+ expect(compile("1 - B", expressionOpts)).toEqual(["-", 1, ['field-id', 2]]);
+ expect(compile("1 - 2", expressionOpts)).toEqual(["-", 1, 2]);
});
it("can handle operator precedence", () => {
- expect(compile("1 + 2 * 3", { fields: mockFields })).toEqual(["+", 1, ["*", 2, 3]]);
- expect(compile("1 * 2 + 3", { fields: mockFields })).toEqual(["+", ["*", 1, 2], 3]);
+ expect(compile("1 + 2 * 3", expressionOpts)).toEqual(["+", 1, ["*", 2, 3]]);
+ expect(compile("1 * 2 + 3", expressionOpts)).toEqual(["+", ["*", 1, 2], 3]);
+ });
+
+ it("can collapse consecutive identical operators", () => {
+ expect(compile("1 + 2 + 3 * 4 * 5", expressionOpts)).toEqual(["+", 1, 2, ["*", 3, 4, 5]]);
});
// quoted field name w/ a space in it
it("can parse a field with quotes and spaces", () => {
- expect(compile("\"Toucan Sam\" + B", { fields: mockFields })).toEqual(["+", ['field-id', 10], ['field-id', 2]]);
+ expect(compile("\"Toucan Sam\" + B", expressionOpts)).toEqual(["+", ['field-id', 10], ['field-id', 2]]);
});
// parentheses / nested parens
it("can parse expressions with parentheses", () => {
- expect(compile("(1 + 2) * 3", { fields: mockFields })).toEqual(["*", ["+", 1, 2], 3]);
- expect(compile("1 * (2 + 3)", { fields: mockFields })).toEqual(["*", 1, ["+", 2, 3]]);
- expect(compile("\"Toucan Sam\" + (A * (B / C))", { fields: mockFields })).toEqual(
+ expect(compile("(1 + 2) * 3", expressionOpts)).toEqual(["*", ["+", 1, 2], 3]);
+ expect(compile("1 * (2 + 3)", expressionOpts)).toEqual(["*", 1, ["+", 2, 3]]);
+ expect(compile("\"Toucan Sam\" + (A * (B / C))", expressionOpts)).toEqual(
["+", ['field-id', 10], ["*", [ 'field-id', 1 ], ["/", [ 'field-id', 2 ], [ 'field-id', 3 ]]]]
);
});
it("can parse aggregation with no arguments", () => {
- expect(compile("Count()", { fields: mockFields })).toEqual(["count"]);
+ expect(compile("Count()", aggregationOpts)).toEqual(["count"]);
});
it("can parse aggregation with argument", () => {
- expect(compile("Sum(A)", { fields: mockFields })).toEqual(["sum", ["field-id", 1]]);
+ expect(compile("Sum(A)", aggregationOpts)).toEqual(["sum", ["field-id", 1]]);
});
it("can parse complex aggregation", () => {
- expect(compile("1 - Sum(A * 2) / Count()", { fields: mockFields })).toEqual(["-", 1, ["/", ["sum", ["*", ["field-id", 1], 2]], ["count"]]]);
+ expect(compile("1 - Sum(A * 2) / Count()", aggregationOpts)).toEqual(["-", 1, ["/", ["sum", ["*", ["field-id", 1], 2]], ["count"]]]);
});
it("should throw exception on invalid input", () => {
- expect(() => compile("1 + ", { fields: mockFields })).toThrow();
+ expect(() => compile("1 + ", expressionOpts)).toThrow();
});
// fks
@@ -69,24 +87,64 @@ describe("lib/expressions/parser", () => {
});
describe("suggest()", () => {
- it("should suggest things after an operator", () => {
- expect(cleanSuggestions(suggest("1 + ", { fields: mockFields.slice(-2) }))).toEqual([
- { type: 'aggregation', text: 'Count(' },
- { type: 'aggregation', text: 'Sum(' },
- { type: 'field', text: '"Toucan Sam"' },
- { type: 'field', text: 'C' },
- { type: 'other', text: '(' },
+ it("should suggest aggregations and metrics after an operator", () => {
+ expect(cleanSuggestions(suggest("1 + ", aggregationOpts))).toEqual([
+ { type: 'aggregations', text: 'Count()' },
+ { type: 'aggregations', text: 'Sum(' },
+ { type: 'metrics', text: 'FooBar()' },
+ { type: 'other', text: ' (' },
+ ]);
+ })
+ it("should suggest fields after an operator", () => {
+ expect(cleanSuggestions(suggest("1 + ", expressionOpts))).toEqual([
+ { type: 'fields', text: '"Toucan Sam" ' },
+ { type: 'fields', text: 'A ' },
+ { type: 'fields', text: 'B ' },
+ { type: 'fields', text: 'C ' },
+ { type: 'other', text: ' (' },
+ ]);
+ })
+ it("should suggest partial matches in aggregation", () => {
+ expect(cleanSuggestions(suggest("1 + C", aggregationOpts))).toEqual([
+ { type: 'aggregations', text: 'Count()' },
]);
})
- it("should suggest partial matches", () => {
- expect(cleanSuggestions(suggest("1 + C", { fields: mockFields.slice(-2) }))).toEqual([
- { type: 'aggregation', text: 'Count(' },
- { type: 'field', text: 'C' }
+ it("should suggest partial matches in expression", () => {
+ expect(cleanSuggestions(suggest("1 + C", expressionOpts))).toEqual([
+ { type: 'fields', text: 'C ' },
]);
})
})
+
+ describe("compile() in syntax mode", () => {
+ it ("should parse source without whitespace into a recoverable syntax tree", () => {
+ const source = "1-Sum(A*2+\"Toucan Sam\")/Count()";
+ const tree = parse(source, aggregationOpts);
+ expect(serialize(tree)).toEqual(source)
+ })
+ xit ("should parse source with metric into a recoverable syntax tree", () => {
+ // FIXME: not preserving parens
+ const source = "Count()+FooBar()";
+ const tree = parse(source, aggregationOpts);
+ expect(serialize(tree)).toEqual(source)
+ })
+ xit ("should parse source with whitespace into a recoverable syntax tree", () => {
+ // FIXME: not preserving whitespace
+ const source = "1 - Sum(A * 2 + \"Toucan Sam\") / Count()";
+ const tree = parse(source, aggregationOpts);
+ expect(serialize(tree)).toEqual(source)
+ })
+ })
});
+function serialize(tree) {
+ if (tree.type === "token") {
+ return tree.text;
+ } else {
+ return tree.children.map(serialize).join("");
+ }
+}
+
function cleanSuggestions(suggestions) {
return _.chain(suggestions).map(s => _.pick(s, "type", "text")).sortBy("text").sortBy("type").value();
}
diff --git a/frontend/test/unit/lib/query.spec.js b/frontend/test/unit/lib/query.spec.js
index 76a6cc07187..2f9aade317f 100644
--- a/frontend/test/unit/lib/query.spec.js
+++ b/frontend/test/unit/lib/query.spec.js
@@ -189,7 +189,7 @@ describe('Query', () => {
filter: []
};
Query.removeBreakout(query, 0);
- expect(query.breakout.length).toBe(0);
+ expect(query.breakout).toBe(undefined);
});
it('should remove sort clauses for the dimension that was removed', () => {
let query = {
diff --git a/package.json b/package.json
index 02db1acd0e5..6dcc966c603 100644
--- a/package.json
+++ b/package.json
@@ -12,7 +12,7 @@
"dependencies": {
"ace-builds": "^1.2.2",
"babel-polyfill": "^6.6.1",
- "chevrotain": "^0.19.0",
+ "chevrotain": "^0.20.0",
"classnames": "^2.1.3",
"color": "^0.11.1",
"crossfilter": "^1.3.12",
diff --git a/yarn.lock b/yarn.lock
index 934c4aa8b03..3904facc61f 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -1444,9 +1444,9 @@ change-emitter@^0.1.2:
version "0.1.2"
resolved "https://registry.yarnpkg.com/change-emitter/-/change-emitter-0.1.2.tgz#6b88ca4d5d864e516f913421b11899a860aee8db"
-chevrotain@^0.19.0:
- version "0.19.0"
- resolved "https://registry.yarnpkg.com/chevrotain/-/chevrotain-0.19.0.tgz#3319bedf0f9426831be870ffa2162be733f4369b"
+chevrotain@^0.20.0:
+ version "0.20.0"
+ resolved "https://registry.yarnpkg.com/chevrotain/-/chevrotain-0.20.0.tgz#b814daf9e15c134bf421ff54b429b818c6648a9e"
chokidar@^1.0.0, chokidar@^1.4.1:
version "1.6.0"
--
GitLab