Skip to content
Snippets Groups Projects
Unverified Commit 274a7bb0 authored by Alexander Kiselev's avatar Alexander Kiselev Committed by GitHub
Browse files

Custom expression editor: use new parser/compiler to generate diagnostics (#19557)

* pratt parser diagnostics

* ported over tests from custom parser branch

* updated error types to support pos/len for error markers

* updated compiler error to removed unnecessary plural

* updated resolver to use new error type and pass through node information

* updated error text in custom column e2e test

* updated diagnostics to use new resolver error class

* update recursive-parser passes to pass through node information

* roll back error text fix in e2e tests

* deleted fuzzing tests (they will be moved into the metabase/expression repo)

* this is weird, setting e2e error text back to "Expected expression"

* rolled back expression editor changes in handleInputBlur and moved diagnoseExpression first to fix exponent rewrite issue

* roll back changes to compileExpression

* update import paths in expressions case class identities get messed up
parent 4097d58c
No related branches found
No related tags found
No related merge requests found
Showing
with 1660 additions and 29 deletions
import { t } from "ttag";
import {
getMBQLName,
parseDimension,
parseMetric,
parseSegment,
} from "metabase/lib/expressions";
import { resolve } from "metabase/lib/expressions/resolver";
import {
parse,
lexify,
compile,
ResolverError,
} from "metabase/lib/expressions/pratt";
import {
useShorthands,
adjustCase,
adjustOptions,
} from "metabase/lib/expressions/recursive-parser";
import { tokenize, TOKEN, OPERATOR } from "metabase/lib/expressions/tokenizer";
import { getMBQLName } from "metabase/lib/expressions";
import { processSource } from "metabase/lib/expressions/process";
// e.g. "COUNTIF(([Total]-[Tax] <5" returns 2 (missing parentheses)
export function countMatchingParentheses(tokens) {
......@@ -53,12 +69,66 @@ export function diagnose(source, startRule, query) {
return { message };
}
const { compileError } = processSource({ source, query, startRule });
try {
return prattCompiler(source, startRule, query);
} catch (err) {
return err;
}
}
function prattCompiler(source, startRule, query) {
const tokens = lexify(source);
const options = { source, startRule, query };
// PARSE
const { root, errors } = parse(tokens, {
throwOnError: false,
...options,
});
if (errors.length > 0) {
return errors[0];
}
function resolveMBQLField(kind, name, node) {
if (!query) {
return [kind, name];
}
if (kind === "metric") {
const metric = parseMetric(name, options);
if (!metric) {
throw new ResolverError(`Unknown Field: ${name}`, node);
}
return ["metric", metric.id];
} else if (kind === "segment") {
const segment = parseSegment(name, options);
if (!segment) {
throw new ResolverError(`Unknown Field: ${name}`, node);
}
return ["segment", segment.id];
} else {
// fallback
const dimension = parseDimension(name, options);
if (!dimension) {
throw new ResolverError(`Unknown Field: ${name}`, node);
}
return dimension.mbql();
}
}
if (compileError) {
return Array.isArray(compileError) && compileError.length > 0
? compileError[0]
: compileError;
// COMPILE
try {
compile(root, {
passes: [
adjustOptions,
useShorthands,
adjustCase,
expr => resolve(expr, startRule, resolveMBQLField),
],
getMBQLName,
});
} catch (err) {
console.warn("compile error", err);
return err;
}
return null;
......
import {
/* ALL_ASTYPES */ ADD,
FIELD,
LOGICAL_AND,
CALL,
EQUALITY,
NUMBER,
LOGICAL_OR,
COMPARISON,
GROUP,
MULDIV_OP,
STRING,
SUB,
NEGATIVE,
LOGICAL_NOT,
IDENTIFIER,
ROOT,
ARG_LIST,
} from "metabase/lib/expressions/pratt/syntax";
import {
assert,
NodeType,
Node,
CompileError,
} from "metabase/lib/expressions/pratt/types";
export type Expr = number | string | ([string, ...Expr[]] & { node?: Node });
export type CompilerPass = (expr: Expr) => Expr;
export interface Options {
getMBQLName(expressionName: string): string | undefined;
passes?: CompilerPass[];
}
type CompileFn = (node: Node, opts: Options) => Expr;
export function compile(node: Node, opts: Options): Expr {
assert(node.type === ROOT, "Must be root node");
if (node.children.length > 1) {
throw new CompileError("Unexpected expression", {
node: node.children[1],
token: node.children[1].token,
});
}
const func = compileUnaryOp(node);
let expr = func(node.children[0], opts);
const { passes = [] } = opts;
for (const pass of passes) {
expr = pass(expr);
}
return expr;
}
// ----------------------------------------------------------------
function compileField(node: Node): Expr {
assert(node.type === FIELD, "Invalid Node Type");
assert(node.token?.text, "Empty field name");
// Slice off the leading and trailing brackets
const name = node.token.text.slice(1, node.token.text.length - 1);
return withNode(["dimension", name], node);
}
function compileIdentifier(node: Node): Expr {
assert(node.type === IDENTIFIER, "Invalid Node Type");
assert(node.token?.text, "Empty token text");
const name = node.token.text;
return withNode(["dimension", name], node);
}
function compileGroup(node: Node, opts: Options): Expr {
assert(node.type === GROUP, "Invalid Node Type");
const func = compileUnaryOp(node);
return func(node.children[0], opts);
}
function compileString(node: Node): Expr {
assert(node.type === STRING, "Invalid Node Type");
assert(typeof node.token?.text === "string", "No token text");
// Slice off the leading and trailing quotes
return node.token.text.slice(1, node.token.text.length - 1);
}
// ----------------------------------------------------------------
function compileLogicalNot(node: Node, opts: Options): Expr {
assert(node.type === LOGICAL_NOT, "Invalid Node Type");
const func = compileUnaryOp(node);
assert(node.token?.text, "Empty token text");
const child = node.children[0];
return withNode(["not", func(child, opts)], node);
}
function compileLogicalAnd(node: Node, opts: Options): Expr {
assert(node.type === LOGICAL_AND, "Invalid Node Type");
assert(node.token?.text, "Empty token text");
const [left, right] = compileInfixOp(node, opts);
return withNode([node.token?.text.toLowerCase(), ...left, ...right], node);
}
function compileLogicalOr(node: Node, opts: Options): Expr {
assert(node.type === LOGICAL_OR, "Invalid Node Type");
assert(node.token?.text, "Empty token text");
const [left, right] = compileInfixOp(node, opts);
return withNode([node.token?.text.toLowerCase(), ...left, ...right], node);
}
function compileComparisonOp(node: Node, opts: Options): Expr {
assert(node.type === COMPARISON, "Invalid Node Type");
const text = node.token?.text;
assert(text, "Empty token text");
const [left, right] = compileInfixOp(node, opts);
return withNode([text, ...left, ...right], node);
}
function compileEqualityOp(node: Node, opts: Options): Expr {
assert(node.type === EQUALITY, "Invalid Node Type");
assert(node.token?.text, "Empty token text");
const [left, right] = compileInfixOp(node, opts);
return withNode([node.token?.text, ...left, ...right], node);
}
// ----------------------------------------------------------------
function compileFunctionCall(node: Node, opts: Options): Expr {
assert(node.type === CALL, "Invalid Node Type");
assert(node.token?.text, "Empty token text");
assert(
node.children[0].type === ARG_LIST,
"First argument must be an arglist",
);
const text = node.token?.text;
const fn = opts.getMBQLName(text.trim().toLowerCase());
return withNode(
[fn ? fn : text, ...compileArgList(node.children[0], opts)],
node,
);
}
function compileArgList(node: Node, opts: Options): Expr[] {
assert(node.type === ARG_LIST, "Invalid Node Type");
return node.children.map(child => {
const func = getCompileFunction(child);
if (!func) {
throw new CompileError("Invalid node type", { node: child });
}
const expr = func(child, opts);
return (expr as any).node ? expr : withNode(expr, child);
});
}
// ----------------------------------------------------------------
function compileNumber(node: Node): Expr {
assert(node.type === NUMBER, "Invalid Node Type");
assert(typeof node.token?.text === "string", "No token text");
try {
return parseFloat(node.token.text);
} catch (err) {
throw new CompileError("Invalid number format", {
node,
token: node.token,
});
}
}
function compileNegative(node: Node, opts: Options): Expr {
assert(node.type === NEGATIVE, "Invalid Node Type");
const func = compileUnaryOp(node);
assert(node.token?.text, "Empty token text");
const child = node.children[0];
if (child.type === NUMBER) {
return -func(child, opts);
}
return withNode(["-", func(child, opts)], node);
}
function compileAdditionOp(node: Node, opts: Options): Expr {
assert(node.type === ADD, "Invalid Node Type");
assert(node.token?.text, "Empty token text");
const [left, right] = compileInfixOp(node, opts);
return withNode([node.token?.text, ...left, ...right], node);
}
function compileMulDivOp(node: Node, opts: Options): Expr {
assert(node.type === MULDIV_OP, "Invalid Node Type");
const text = node.token?.text;
assert(text, "Empty token text");
const [left, right] = compileInfixOp(node, opts);
return withNode([text, ...left, ...right], node);
}
function compileSubtractionOp(node: Node, opts: Options): Expr {
assert(node.type === SUB, "Invalid Node Type");
assert(node.token?.text, "Empty token text");
const [left, right] = compileInfixOp(node, opts);
return withNode([node.token?.text, ...left, ...right], node);
}
// ----------------------------------------------------------------
function compileUnaryOp(node: Node) {
if (node.children.length > 1) {
throw new CompileError("Unexpected expression", {
node: node.children[1],
token: node.children[1].token,
});
} else if (node.children.length === 0) {
throw new CompileError("Expected expression", { node, token: node.token });
}
const func = getCompileFunction(node.children[0]);
if (!func) {
throw new CompileError("Invalid node type", {
node: node.children[0],
token: node.children[0].token,
});
}
return func;
}
function compileInfixOp(node: Node, opts: Options) {
if (node.children.length > 2) {
throw new CompileError("Unexpected expression", {
node: node.children[2],
token: node.children[2].token,
});
} else if (node.children.length === 0) {
throw new CompileError("Expected expression", { node, token: node.token });
}
const leftFn = getCompileFunction(node.children[0]);
if (!leftFn) {
throw new CompileError("Invalid node type", {
node: node.children[0],
token: node.children[0].token,
});
}
const rightFn = getCompileFunction(node.children[1]);
if (!rightFn) {
throw new CompileError("Invalid node type", {
node: node.children[1],
token: node.children[1].token,
});
}
const text = node.token?.text;
let left: any = leftFn(node.children[0], opts);
if (Array.isArray(left) && left[0]?.toUpperCase() === text?.toUpperCase()) {
const [_, ...args] = left;
left = args;
} else {
left = [left];
}
let right: any = rightFn(node.children[1], opts);
right = [right];
return [left, right];
}
function withNode<T>(expr: T, node: Node): T {
if (typeof expr === "object") {
Object.defineProperty(expr, "node", {
writable: false,
enumerable: false,
value: node,
});
}
return expr;
}
// ----------------------------------------------------------------
const COMPILE = new Map<NodeType, CompileFn>([
[FIELD, compileField],
[ADD, compileAdditionOp],
[LOGICAL_AND, compileLogicalAnd],
[CALL, compileFunctionCall],
[EQUALITY, compileEqualityOp],
[NUMBER, compileNumber],
[LOGICAL_NOT, compileLogicalNot],
[NEGATIVE, compileNegative],
[LOGICAL_OR, compileLogicalOr],
[COMPARISON, compileComparisonOp],
[GROUP, compileGroup],
[MULDIV_OP, compileMulDivOp],
[STRING, compileString],
[SUB, compileSubtractionOp],
[IDENTIFIER, compileIdentifier],
]);
function getCompileFunction(node: Node): (node: Node, opts: Options) => Expr {
const func = COMPILE.get(node.type);
if (!func) {
throw new CompileError("Invalid node type", { node, token: node.token });
}
return func;
}
export * from "metabase/lib/expressions/pratt/parser";
export * from "metabase/lib/expressions/pratt/compiler";
export * from "metabase/lib/expressions/pratt/syntax";
export * from "metabase/lib/expressions/pratt/types";
import {
ADD,
ARG_LIST,
BAD_TOKEN,
CALL,
COMMA,
COMPARISON,
END_OF_INPUT,
EQUALITY,
FIELD,
GROUP,
GROUP_CLOSE,
IDENTIFIER,
LOGICAL_AND,
LOGICAL_NOT,
LOGICAL_OR,
MULDIV_OP,
NEGATIVE,
NUMBER,
ROOT,
STRING,
SUB,
WS,
} from "metabase/lib/expressions/pratt/syntax";
import {
assert,
CompileError,
NodeType,
Token,
Node,
Hooks,
} from "metabase/lib/expressions/pratt/types";
import { tokenize, TOKEN, OPERATOR } from "metabase/lib/expressions/tokenizer";
interface ParserOptions {
hooks?: Hooks;
maxIterations?: number;
throwOnError?: boolean;
}
interface ParserResult {
root: Node;
errors: CompileError[];
}
export function lexify(expression: string) {
const lexs: Token[] = [];
const { tokens, errors } = tokenize(expression);
if (errors && errors.length > 0) {
errors.forEach(error => {
const { pos } = error;
lexs.push({ type: BAD_TOKEN, text: expression[pos], length: 1, pos });
});
}
let start = 0;
for (let i = 0; i < tokens.length; ++i) {
const token = tokens[i];
if (start < token.start) {
lexs.push({
type: WS,
text: expression.slice(start, token.start),
length: token.start - start,
pos: start,
});
}
start = token.end;
let text = expression.slice(token.start, token.end);
const pos = token.start;
let length = token.end - token.start;
let type = BAD_TOKEN;
switch (token.type) {
case TOKEN.Number:
type = NUMBER;
break;
case TOKEN.String:
type = STRING;
break;
case TOKEN.Identifier:
type = text[0] === "[" ? FIELD : IDENTIFIER;
break;
case TOKEN.Operator:
switch (token.op) {
case OPERATOR.Comma:
type = COMMA;
break;
case OPERATOR.OpenParenthesis:
type = GROUP;
break;
case OPERATOR.CloseParenthesis:
type = GROUP_CLOSE;
break;
case OPERATOR.Plus:
type = ADD;
break;
case OPERATOR.Minus:
type = SUB;
break;
case OPERATOR.Star:
case OPERATOR.Slash:
type = MULDIV_OP;
break;
case OPERATOR.Equal:
case OPERATOR.NotEqual:
type = EQUALITY;
break;
case OPERATOR.LessThan:
case OPERATOR.GreaterThan:
case OPERATOR.LessThanEqual:
case OPERATOR.GreaterThanEqual:
type = COMPARISON;
break;
case OPERATOR.Not:
type = LOGICAL_NOT;
break;
case OPERATOR.And:
type = LOGICAL_AND;
break;
case OPERATOR.Or:
type = LOGICAL_OR;
break;
default:
break;
}
break;
}
if (type === IDENTIFIER) {
const next = tokens[i + 1];
if (
next &&
next.type === TOKEN.Operator &&
next.op === OPERATOR.OpenParenthesis
) {
type = CALL;
length = next.start - token.start;
text = expression.slice(token.start, next.start);
start = next.start;
}
}
lexs.push({ type, text, length, pos });
}
// This simplifies the parser
lexs.push({
type: END_OF_INPUT,
text: "\n",
length: 1,
pos: expression.length,
});
return lexs.sort((a, b) => a.pos - b.pos);
}
export function parse(tokens: Token[], opts: ParserOptions = {}): ParserResult {
const { maxIterations = 1000000, hooks = {}, throwOnError = false } = opts;
const errors: CompileError[] = [];
let counter = 0;
const root = createASTNode(null, null, ROOT, counter);
root.isRoot = true;
let node = root;
hooks.onCreateNode?.(tokens[0], node);
for (
let index = 0;
index < tokens.length && counter < maxIterations;
index++
) {
const token = tokens[index];
hooks.onIteration?.(token, node);
if (token.type.skip) {
hooks.onSkipToken?.(token, node);
continue;
}
if (token.type === BAD_TOKEN) {
const err = new CompileError(`Unexpected token "${token.text}"`, {
node,
token,
});
hooks.onBadToken?.(token, node, err);
if (throwOnError) {
throw err;
}
errors.push(err);
// If we don't throw on error, we skip the bad token
continue;
}
if (node.complete) {
// If a node has received all the children it expects, it's time to figure
// out whether it needs to be reparented. This is the core of the
// our solution to the predence issue. By default, we can expect the node
// to go to its parent but if the next token has a higher precedence (like
// `*` over `+`), it might take the node instead.
assert(
node.parent,
"Marked a node complete without placing it with a parent",
);
// This is the core of the precedence climbing logic. If a higher priority
// token is encountered, shouldReparent will return true and the new node
// we created for the token will "take" the current node
if (shouldReparent(node.parent.type, token.type)) {
node.parent = createASTNode(
token,
node.parent,
getASType(token.type, node.parent.type),
counter,
);
hooks.onReparentNode?.(token, node);
} else {
// If we don't need to reparent, we decrement the token index. This is
// because we iterate several times for each node, first to create it
// and then to check if it is completed.
index--;
}
// Place the node in its parent children and get the next "active" node
// which is node.parent
node = place(node, errors, opts);
if (node.children.length === node.type.expectedChildCount) {
node.complete = true;
hooks.onCompleteNode?.(token, node);
}
} else if (token.type.isTerminator) {
hooks.onTerminatorToken?.(token, node);
// Terminator tokens like `]`, `)` or end of input will complete a node if
// they match the type's `requiresTerminator`
if (node.type.requiresTerminator === token.type) {
node.complete = true;
hooks.onCompleteNode?.(token, node);
} else if (node.type.ignoresTerminator.indexOf(token.type) === -1) {
// If the current token isn't in the list of the AST type's ignored
// tokens and it's not the terminator the current node requires, we'll
// throw an error
const err = new CompileError(`Expected expression`, { node, token });
hooks.onUnexpectedTerminator?.(token, node, err);
if (throwOnError) {
throw err;
}
errors.push(err);
if (token.type === END_OF_INPUT) {
// We complete and reparent/place the final node by running the for
// loop one last time
if (!node.complete) {
node.complete = true;
hooks.onCompleteNode?.(token, node);
index--;
}
}
}
} else if (token.type.leftOperands !== 0) {
// Subtraction is a special case because it might actually be negation
if (token.type === SUB) {
node = createASTNode(token, node, NEGATIVE, counter);
hooks.onCreateNode?.(token, node);
} else {
const err = new CompileError(`Expected expression`, {
token,
});
hooks.onMissinChildren?.(token, node, err);
if (throwOnError) {
throw err;
}
errors.push(err);
}
} else {
// Create the AST node. It will be marked as complete if the node doesn't
// expect any children (like a literal or identifier)
node = createASTNode(
token,
node,
getASType(token.type, node.type),
counter,
);
hooks.onCreateNode?.(token, node);
}
counter += 1;
}
if (counter >= maxIterations) {
throw new Error("Reached max number of iterations");
}
const childViolation = ROOT.checkChildConstraints(root);
if (childViolation !== null) {
const err = new CompileError("Unexpected token", {
node: root,
...childViolation,
});
hooks.onChildConstraintViolation?.(node, err);
if (throwOnError) {
throw err;
}
errors.push(err);
}
return { root, errors };
}
function createASTNode(
token: Token | null,
parent: Node | null,
type: NodeType,
counter: number,
): Node {
return {
type,
children: [],
complete: type.expectedChildCount === 0,
parent,
token,
resolvedType: type.resolvesAs ? type.resolvesAs : counter,
};
}
function place(node: Node, errors: CompileError[], opts: ParserOptions) {
const { hooks = {}, throwOnError = false } = opts;
const { type, parent } = node;
const childViolation = type.checkChildConstraints(node);
if (childViolation !== null) {
const err = new CompileError("Unexpected token", {
node,
...childViolation,
});
hooks.onChildConstraintViolation?.(node, err);
if (throwOnError) {
throw err;
}
errors.push(err);
}
assert(parent, "Tried to place a node without a parent", node);
parent.children.push(node);
hooks.onPlaceNode?.(node, parent);
return parent;
}
function shouldReparent(leftType: NodeType, rightType: NodeType) {
// If the right node doesn't have any left operands like a literal or
// identifier, then it can't become the parent of the left node anyway
if (rightType.leftOperands === 0) {
return false;
} else {
return rightType.precedence > leftType.precedence;
}
}
export function getASType(type: NodeType, parentType: NodeType) {
if (type === GROUP) {
// A list of function call arguments is first interpreted as a GROUP, then
// reinterpreted as an ARG_LIST if its the child of a CALL
if (parentType === CALL) {
return ARG_LIST;
}
}
return type;
}
import { assert, NodeType, Node } from "metabase/lib/expressions/pratt/types";
/*
* This file specifies most of the syntax for the Metabase handwritten custom
* expression parser. The rest is contained in the parser special cases
*
* The structure of this file:
* 1. Declare all of the ASTypes (types of nodes that appear in the abstract
* syntax tree)
* 2. Associate various properties with those ASTypes that determine the
* overall shape of the grammar (# of operands, scope info, etc.)
* 3. Declare more grammatical rules as functions/generate them from tables
* (parent/child Type constraints, operator precedence tiers)
* 4. Export getASType(), which the parser uses to override the Type guesses
* that the lexer makes based on contextual information
*/
export const FIELD = {} as NodeType;
export const ADD = {} as NodeType;
export const LOGICAL_AND = {} as NodeType;
export const ARG_LIST = {} as NodeType;
export const BAD_TOKEN = {} as NodeType;
export const CALL = {} as NodeType;
export const COMMA = {} as NodeType;
export const END_OF_INPUT = {} as NodeType;
export const EQUALITY = {} as NodeType;
export const NUMBER = {} as NodeType;
export const LOGICAL_NOT = {} as NodeType;
export const NEGATIVE = {} as NodeType;
export const LOGICAL_OR = {} as NodeType;
export const COMPARISON = {} as NodeType;
export const GROUP = {} as NodeType;
export const GROUP_CLOSE = {} as NodeType;
export const ROOT = {} as NodeType;
export const MULDIV_OP = {} as NodeType;
export const STRING = {} as NodeType;
export const SUB = {} as NodeType;
export const IDENTIFIER = {} as NodeType;
export const WS = {} as NodeType;
function operand(leftOperands: number, rightOperands: number) {
return {
leftOperands,
rightOperands,
expectedChildCount: leftOperands + rightOperands,
};
}
function setAttributes(...syntaxRules: [Partial<NodeType>, NodeType[]][]) {
for (const [values, types] of syntaxRules) {
for (const type of types) {
Object.assign(type, values);
}
}
}
const ALL_NODES = [
ADD,
LOGICAL_AND,
ARG_LIST,
BAD_TOKEN,
CALL,
COMMA,
END_OF_INPUT,
EQUALITY,
NUMBER,
NEGATIVE,
LOGICAL_NOT,
LOGICAL_OR,
COMPARISON,
GROUP,
GROUP_CLOSE,
ROOT,
MULDIV_OP,
STRING,
SUB,
FIELD,
IDENTIFIER,
WS,
];
// Set default values for AST node attributes
setAttributes([
{
skip: false,
leftOperands: 0,
rightOperands: 0,
expectedChildCount: 0,
checkChildConstraints: () => null,
requiresTerminator: null,
ignoresTerminator: [],
isTerminator: false,
precedence: -Infinity,
resolvesAs: null,
expectedTypes: null,
},
ALL_NODES,
]);
setAttributes(
// Prefix Operators
[operand(0, 1), [CALL, NEGATIVE, LOGICAL_NOT]],
// Infix Operators
[
operand(1, 1),
[MULDIV_OP, ADD, SUB, COMPARISON, EQUALITY, LOGICAL_AND, LOGICAL_OR],
],
// Open Expressions (various paren types, blocks, etc.) and their terminators
[{ expectedChildCount: Infinity }, [ARG_LIST, ROOT, GROUP]],
[{ ignoresTerminator: [COMMA] }, [ARG_LIST]],
[{ requiresTerminator: END_OF_INPUT }, [ROOT]],
[{ requiresTerminator: GROUP_CLOSE }, [ARG_LIST, GROUP]],
[{ isTerminator: true }, [COMMA, END_OF_INPUT, GROUP_CLOSE]],
// Skip whitespace
[{ skip: true }, [WS]],
// Known types
[{ resolvesAs: "string" }, [STRING]],
[{ resolvesAs: "number" }, [ADD, NUMBER, NEGATIVE, MULDIV_OP, SUB]],
[
{ resolvesAs: "boolean" },
[LOGICAL_AND, EQUALITY, LOGICAL_NOT, LOGICAL_OR, COMPARISON],
],
// Expected types
[
{ expectedTypes: ["number"] },
[ADD, NUMBER, NEGATIVE, MULDIV_OP, SUB, COMPARISON],
],
[{ expectedTypes: ["boolean"] }, [LOGICAL_NOT, LOGICAL_AND, LOGICAL_OR]],
[{ expectedTypes: ["boolean", "number", "string"] }, [EQUALITY]],
);
/*
* Child constraints govern how many children an AST node can have and where
* thare placed relative to the node.
*
* These are syntax rules, rather than semantic ones, since that is handled
* later by a different pass. i.e. LOGICAL_AND and LOGICAL_OR rules don't check
* that the right and left side resolve to boolean types.
*
* `checkChildConstraints` returns an object with diagnostic information if
* there is a constraint violation, null otherwise.
*/
function childConstraintByPosition(...positions: NodeType[][]) {
return (node: Node) => {
for (let i = 0; i < positions.length; i++) {
if (!node.children[i]) {
return { position: i, expected: positions };
} else if (!positions[i].includes(node.children[i].type)) {
return { position: i, child: node.children[i], expected: positions };
}
}
return null;
};
}
LOGICAL_NOT.checkChildConstraints = childConstraintByPosition([
FIELD,
IDENTIFIER,
LOGICAL_NOT,
LOGICAL_OR,
LOGICAL_AND,
COMPARISON,
EQUALITY,
CALL,
GROUP,
NEGATIVE,
NUMBER,
STRING,
ADD,
SUB,
MULDIV_OP,
]);
NEGATIVE.checkChildConstraints = childConstraintByPosition([
NUMBER,
FIELD,
IDENTIFIER,
NEGATIVE,
CALL,
GROUP,
ADD,
SUB,
MULDIV_OP,
LOGICAL_NOT,
LOGICAL_OR,
LOGICAL_AND,
COMPARISON,
STRING,
]);
CALL.checkChildConstraints = childConstraintByPosition([ARG_LIST]);
function anyChildConstraint(...acceptableTypes: NodeType[]) {
return (node: Node) => {
for (const child of node.children) {
if (!acceptableTypes.includes(child.type)) {
return { child };
}
}
return null;
};
}
ROOT.checkChildConstraints = anyChildConstraint(
FIELD,
ADD,
LOGICAL_AND,
CALL,
EQUALITY,
NUMBER,
NEGATIVE,
LOGICAL_NOT,
MULDIV_OP,
LOGICAL_OR,
COMPARISON,
GROUP,
STRING,
SUB,
IDENTIFIER,
);
/*
* This defines the operator precedence in order from highest priority to lowest
* priority. When a node with a higher precedence is encountered, the node with
* the lower precedence is "reparented" into the higher node.
*/
[
[CALL],
[FIELD],
[NEGATIVE],
[MULDIV_OP],
[ADD, SUB],
[EQUALITY, COMPARISON],
[LOGICAL_NOT],
[LOGICAL_AND],
[LOGICAL_OR],
[IDENTIFIER],
].forEach((tier, precedence, tiers) => {
for (const type of tier) {
type.precedence = tiers.length - precedence;
}
});
// Give each node
for (const [key, value] of Object.entries(ALL_NODES)) {
value.name = key;
}
SUB.name = "SUBTRACT";
WS.name = "WHITESPACE";
import { isProduction } from "metabase/env";
export const VOID = Symbol("Unknown Type");
export type VariableKind =
| "dimension"
| "segment"
| "aggregation"
| "expression";
export type Type = VariableKind | "string" | "number" | "boolean";
export type VariableId = number;
export interface Token {
type: NodeType;
text: string;
length: number;
pos: number;
}
export interface Node {
type: NodeType;
children: Node[];
complete: boolean;
resolvedType: Type | VariableId;
parent: Node | null;
token: Token | null;
isRoot?: boolean;
}
export interface NodeType {
name?: string;
// Should the parser ignore this sort of token entirely (whitespace)
skip: boolean;
// Number of operands to expect for this node on the left side
leftOperands: number;
// Number of operands to expect for this node on the right side
rightOperands: number;
// Maximum number of children before this node is considered complete. May be
// `Infinity` for nodes lik ARG_LIST, or number of left+right operands
expectedChildCount: number;
// Check child constraints
checkChildConstraints: (
node: Node,
) => { position?: number; child?: Node } | null;
// For open expressions, this is the AST type of tokens that close the
// expression (e.g. GROUP_CLOSE for GROUP).
requiresTerminator: NodeType | null;
// For open expressions, this is a list of AST types that should be considered
// a "separator" (e.g. COMMA for ARG_LIST).
ignoresTerminator: NodeType[];
// Does this token type terminate the current expression (unless exempted by
// .ignoresTerminator)?
isTerminator: boolean;
// The precedence to use for operator parsing conflicts. Higher wins.
precedence: number;
// The type this node resolves to, if it can be deduced early on. If null, the
// parser assigns an integer value for substitutions instead
resolvesAs: Type | null;
// The expectedType of the child nodes
expectedTypes: Type[] | null;
}
type HookFn = (token: Token, node: Node) => void;
type HookErrFn = (token: Token, node: Node, err: CompileError) => void;
type NodeErrFn = (node: Node, err: CompileError) => void;
export interface Hooks {
onIteration?: HookFn;
onCreateNode?: HookFn;
onPlaceNode?: (node: Node, parent: Node) => void;
onSkipToken?: HookFn;
onReparentNode?: HookFn;
onCompleteNode?: HookFn;
onTerminatorToken?: HookFn;
onBadToken?: HookErrFn;
onUnexpectedTerminator?: HookErrFn;
onMissinChildren?: HookErrFn;
onChildConstraintViolation?: NodeErrFn;
}
/*
* This class helps anything that handles parser errors to use instanceof to
* easily distinguish between compilation error exceptions and exceptions due to
* bugs
*/
export abstract class ExpressionError extends Error {
abstract get pos(): number | null;
abstract get len(): number | null;
}
export class CompileError extends ExpressionError {
constructor(message: string, private data: any) {
super(message);
}
get pos(): number | null {
return this.data?.token?.pos ?? null;
}
get len(): number | null {
return this.data?.token?.len ?? null;
}
}
export class ResolverError extends ExpressionError {
constructor(message: string, private node: Node) {
super(message);
}
get pos(): number | null {
return this.node?.token?.pos ?? null;
}
get len(): number | null {
return this.node?.token?.length ?? null;
}
}
export class AssertionError extends Error {
data?: any;
constructor(message: string, data?: any) {
super(`Assertion failed: ${message}`);
this.data = data;
}
}
export function assert(
condition: any,
msg: string,
data?: any,
): asserts condition {
if (isProduction) {
if (!condition) {
throw new AssertionError(msg, data || {});
}
}
}
......@@ -210,12 +210,25 @@ function recursiveParse(source) {
const modify = (node, transform) => {
if (Array.isArray(node)) {
const [operator, ...operands] = node;
return transform([
operator,
...operands.map(sub => modify(sub, transform)),
]);
return withAST(
transform([operator, ...operands.map(sub => modify(sub, transform))]),
node,
);
}
return transform(node);
return withAST(transform(node), node);
};
const withAST = (result, expr) => {
// If this expression comes from the compiler, an object property
// containing the parent AST node will be included for errors
if (expr.node && typeof result.node === "undefined") {
Object.defineProperty(result, "node", {
writable: false,
enumerable: false,
value: expr.node,
});
}
return result;
};
const NEGATIVE_FILTER_SHORTHANDS = {
......@@ -233,7 +246,7 @@ export const useShorthands = tree =>
const [fn, ...params] = operand;
const shorthand = NEGATIVE_FILTER_SHORTHANDS[fn];
if (shorthand) {
return [shorthand, ...params];
return withAST([shorthand, ...params], node);
}
}
}
......@@ -259,7 +272,7 @@ export const adjustOptions = tree =>
operands.pop();
operands.push({ "include-current": true });
}
return [operator, ...operands];
return withAST([operator, ...operands], node);
}
}
}
......@@ -282,9 +295,9 @@ export const adjustCase = tree =>
}
if (operands.length > 2 * pairCount) {
const defVal = operands[operands.length - 1];
return [operator, pairs, { default: defVal }];
return withAST([operator, pairs, { default: defVal }], node);
}
return [operator, pairs];
return withAST([operator, pairs], node);
}
}
return node;
......
import { ngettext, msgid, t } from "ttag";
import { OPERATOR as OP } from "./tokenizer";
import { MBQL_CLAUSES } from "./index";
import { OPERATOR as OP } from "metabase/lib/expressions/tokenizer";
import { ResolverError } from "metabase/lib/expressions/pratt/types";
import { MBQL_CLAUSES } from "metabase/lib/expressions";
const FIELD_MARKERS = ["dimension", "segment", "metric"];
const LOGICAL_OPS = [OP.Not, OP.And, OP.Or];
......@@ -57,7 +58,7 @@ export function resolve(expression, type = "expression", fn = undefined) {
if (FIELD_MARKERS.includes(op)) {
const kind = MAP_TYPE[type] || "dimension";
const [name] = operands;
return fn ? fn(kind, name) : [kind, name];
return fn ? fn(kind, name, expression.node) : [kind, name];
}
let operandType = null;
......@@ -69,7 +70,10 @@ export function resolve(expression, type = "expression", fn = undefined) {
operandType = "expression";
const [firstOperand] = operands;
if (typeof firstOperand === "number" && !Array.isArray(firstOperand)) {
throw new Error(t`Expecting field but found ${firstOperand}`);
throw new ResolverError(
t`Expecting field but found ${firstOperand}`,
expression.node,
);
}
} else if (op === "concat") {
operandType = "expression";
......@@ -78,7 +82,10 @@ export function resolve(expression, type = "expression", fn = undefined) {
} else if (op === "case") {
const [pairs, options] = operands;
if (pairs.length < 1) {
throw new Error(t`CASE expects 2 arguments or more`);
throw new ResolverError(
t`CASE expects 2 arguments or more`,
expression.node,
);
}
const resolvedPairs = pairs.map(([tst, val]) => [
......@@ -105,12 +112,13 @@ export function resolve(expression, type = "expression", fn = undefined) {
const clause = findMBQL(op);
if (!clause) {
throw new Error(t`Unknown function ${op}`);
throw new ResolverError(t`Unknown function ${op}`, expression.node);
}
const { displayName, args, multiple, hasOptions } = clause;
if (!isCompatible(type, clause.type)) {
throw new Error(
throw new ResolverError(
t`Expecting ${type} but found function ${displayName} returning ${clause.type}`,
expression.node,
);
}
if (!multiple) {
......@@ -122,12 +130,13 @@ export function resolve(expression, type = "expression", fn = undefined) {
operands.length < expectedArgsLength ||
operands.length > maxArgCount
) {
throw new Error(
throw new ResolverError(
ngettext(
msgid`Function ${displayName} expects ${expectedArgsLength} argument`,
`Function ${displayName} expects ${expectedArgsLength} arguments`,
expectedArgsLength,
),
expression.node,
);
}
}
......
......@@ -253,10 +253,7 @@ export default class ExpressionEditorTextfield extends React.Component {
this.clearSuggestions();
const { query, startRule } = this.props;
const { source } = this.state;
const errorMessage = diagnose(source, startRule, query);
const errorMessage = this.diagnoseExpression();
this.setState({ errorMessage });
// whenever our input blurs we push the updated expression to our parent if valid
......@@ -315,6 +312,15 @@ export default class ExpressionEditorTextfield extends React.Component {
return expression;
}
diagnoseExpression() {
const { source } = this.state;
if (!source || source.length === 0) {
return { message: "Empty expression" };
}
const { query, startRule } = this.props;
return diagnose(source, startRule, query);
}
commitExpression() {
const { query, startRule } = this.props;
const { source } = this.state;
......
import {
lexify,
parse,
compile as newCompile,
Expr,
} from "metabase/lib/expressions/pratt";
import { resolve } from "metabase/lib/expressions/resolver";
import { getMBQLName } from "metabase/lib/expressions/config";
import {
parse as oldParser,
useShorthands,
adjustCase,
adjustOptions,
} from "metabase/lib/expressions/recursive-parser";
import { generateExpression } from "../generator";
type Type = "expression" | "boolean";
interface Opts {
throwOnError?: boolean;
resolverPass?: boolean;
}
export function compile(source: string, type: Type, opts: Opts = {}) {
const { throwOnError } = opts;
const passes = [adjustOptions, useShorthands, adjustCase];
return newCompile(
parse(lexify(source), {
throwOnError,
}).root,
{
passes: opts.resolverPass
? [...passes, expr => resolve(expr, type, mockResolve as any)]
: passes,
getMBQLName,
},
);
}
export function mockResolve(kind: any, name: string): Expr {
return ["dimension", name];
}
export function oracle(source: string, type: Type) {
let mbql = null;
try {
mbql = oldParser(source);
} catch (e) {
let err = e as any;
if (err.length && err.length > 0) {
err = err[0];
if (typeof err.message === "string") {
err = err.message;
}
}
throw err;
}
return resolve(mbql, type, mockResolve as any);
}
export function compare(
source: string,
type: Type,
opts: Opts = {},
): { oracle: any; compiled: any } {
const _oracle = oracle(source, type);
const compiled = compile(source, type, opts);
return { oracle: _oracle, compiled };
}
export function compareSeed(
seed: number,
type: Type,
opts: Opts = {},
): { oracle: any; compiled: any } {
const { expression } = generateExpression(seed, type);
const _oracle = oracle(expression, type);
const compiled = compile(expression, type, opts);
return { oracle: _oracle, compiled };
}
import { compare, compile } from "./common";
describe("metabase/lib/expressions/compiler", () => {
function expr(
source: string,
opts: { throwOnError?: boolean; resolverPass?: boolean } = {},
) {
const { throwOnError = true, resolverPass } = opts;
return compile(source, "expression", { throwOnError, resolverPass });
}
describe("(for an expression)", () => {
it("should compile literals", () => {
expect(expr("42")).toEqual(42);
expect(expr("'Universe'")).toEqual("Universe");
});
/// TODO: Fix w/ some type info
it("should compile dimensions", () => {
expect(expr("[Price]")).toEqual(["dimension", "Price"]);
expect(expr("([X])")).toEqual(["dimension", "X"]);
});
it("should compile arithmetic operations", () => {
expect(expr("1+2")).toEqual(["+", 1, 2]);
expect(expr("3-4")).toEqual(["-", 3, 4]);
expect(expr("5*6")).toEqual(["*", 5, 6]);
expect(expr("7/8")).toEqual(["/", 7, 8]);
expect(expr("-(1+2)")).toEqual(["-", ["+", 1, 2]]);
});
it("should compile comparisons", () => {
expect(expr("1<2")).toEqual(["<", 1, 2]);
expect(expr("3>4")).toEqual([">", 3, 4]);
expect(expr("5<=6")).toEqual(["<=", 5, 6]);
expect(expr("7>=8")).toEqual([">=", 7, 8]);
expect(expr("9=9")).toEqual(["=", 9, 9]);
expect(expr("9!=0")).toEqual(["!=", 9, 0]);
});
it("should logical operators", () => {
expect(expr("7 or 8")).toEqual(["or", 7, 8]);
expect(expr("7 and 8")).toEqual(["and", 7, 8]);
expect(expr("7 and Size")).toEqual(["and", 7, ["dimension", "Size"]]);
expect(expr("NOT (7 and Size)")).toEqual([
"not",
["and", 7, ["dimension", "Size"]],
]);
});
it("should handle parenthesized expression", () => {
expect(expr("(42)")).toEqual(42);
expect(expr("-42")).toEqual(-42);
expect(expr("-(42)")).toEqual(["-", 42]);
expect(expr("((43))")).toEqual(43);
expect(expr("('Universe')")).toEqual("Universe");
expect(expr("(('Answer'))")).toEqual("Answer");
expect(expr("(1+2)")).toEqual(["+", 1, 2]);
expect(expr("(1+2)/3")).toEqual(["/", ["+", 1, 2], 3]);
expect(expr("4-(5*6)")).toEqual(["-", 4, ["*", 5, 6]]);
expect(expr("func_name(5*6, 4-3)")).toEqual([
"func_name",
["*", 5, 6],
["-", 4, 3],
]);
});
});
describe("Should match the old compiler", () => {
it("Seed 59793: NOT NOT [p]<0", () => {
expect(expr("NOT NOT [p] < 0")).toEqual([
"not",
["not", ["<", ["dimension", "p"], 0]],
]);
});
it("Seed 59809: NOT ( ( [gG9_r]) ) >=( [__] )", () => {
expect(expr("NOT ( ( [gG9_r]) ) >=( [__] )")).toEqual([
"not",
[">=", ["dimension", "gG9_r"], ["dimension", "__"]],
]);
});
it(`Seed 10099: CONtAinS ( [OF4wuV], SUbstriNG("_", -872096.613705, lENGtH("s Mfg7" ) ) )`, () => {
const {
oracle,
compiled,
} = compare(
`CONtAinS ( [OF4wuV], SUbstriNG("_", -872096.613705, lENGtH("s Mfg7" ) ) )`,
"boolean",
{ throwOnError: true, resolverPass: false },
);
expect(compiled).toEqual(oracle);
});
it(`Seed 10092: NOT ( NOT (isNUll ([T0q → n_M_O])))`, () => {
const { oracle, compiled } = compare(
` NOT ( NOT (isNUll ([T0q → n_M_O])))`,
"expression",
{
throwOnError: true,
resolverPass: false,
},
);
expect(compiled).toEqual(oracle);
});
it(`Seed 10082: SUbstriNg( cOncat("BaK2 ", [__m_4], rTrim(coNcAt (replACE ([Av5Wtbz], regeXextRACt( [_1I → g], "H NVB84_"), rEGexextract ( [__8], " _ 2" ) ) , SUbStRiNG("qb0 ", (power( LeNgTh ( rtrim ( "YyCe_2" )) * 0e+77, 1 ) ), 1 + 0e-54 / 374719e-64) , cOncaT( " F9 _O", "_a5_", " 5 _U_ ", " bE", rEPlACe (BXj3O, " ", [Z → X9]) ) ) ) ), (1), log ( 1E-26 ) ) `, () => {
const {
oracle,
compiled,
} = compare(
`SUbstriNg( cOncat("BaK2 ", [__m_4], rTrim(coNcAt (replACE ([Av5Wtbz], regeXextRACt( [_1I → g], "H NVB84_"), rEGexextract ( [__8], " _ 2" ) ) , SUbStRiNG("qb0 ", (power( LeNgTh ( rtrim ( "YyCe_2" )) * 0e+77, 1 ) ), 1 + 0e-54 / 374719e-64) , cOncaT( " F9 _O", "_a5_", " 5 _U_ ", " bE", rEPlACe (BXj3O, " ", [Z → X9]) ) ) ) ), (1), log ( 1E-26 ) )`,
"expression",
{ throwOnError: true, resolverPass: false },
);
expect(compiled).toEqual(oracle);
});
it(`Seed 57808: (( Abs ( (exP( cEil( - 1e+48) )) ) * -1e31 ) * ( poWeR( (( - 0e+67) *lengTh ( "8" ) ) , ( -1 ) *lengTh( "Q P2c n" ) / powEr(1, N) ) ) )`, () => {
const {
oracle,
compiled,
} = compare(
`(( Abs ( (exP( cEil( - 1e+48) )) ) * -1e31 ) * ( poWeR( (( - 0e+67) *lengTh ( "8" ) ) , ( -1 ) *lengTh( "Q P2c n" ) / powEr(1, N) ) ) )`,
"expression",
{ throwOnError: true, resolverPass: false },
);
expect(compiled).toEqual(oracle);
});
// Checks that `x - (y - z)` doesn't get merged into `x - y -z`
it(`seed 10144: eNDsWith( "NTP", replacE( [V2FFf → r_8ZFu], coalescE(repLACe([cf → l], sUbStriNg ( Replace( [A], "L ", "b"), coAlEsCE(953925E-38, 307355.510173e+32 ), pOwEr (1e+15, 0 )) , caSe ( IsEMpTy( [_ → _3H_6b]) , cOncat ("n_F e_B n" ) , isEmptY([E → _3R6p6_]), conCat(" D 2h", " 4 9u ", "A_9_M_9_", " q _")) ) , RegExeXtRact ([_PI9], "K43s 6") ) , sUBstriNg (CaSE (intervAl( [_OU9c], - 632269.595767E-79, CoalESce ("u__0_71", "c ")), CoalesCe( suBstriNG ( "XPHC0 li_", 500924.700063e-10, 341369) )), - 1E+47 - (424024.827478-1 ), - 1) ) )`, () => {
const {
oracle,
compiled,
} = compare(
`(( Abs ( (exP( cEil( - 1e+48) )) ) * -1e31 ) * ( poWeR( (( - 0e+67) *lengTh ( "8" ) ) , ( -1 ) *lengTh( "Q P2c n" ) / powEr(1, N) ) ) )`,
"expression",
{ throwOnError: true, resolverPass: false },
);
expect(compiled).toEqual(oracle);
});
it(`Merging subtraction`, () => {
const { oracle, compiled } = compare(`1 - (0 - -10)`, "expression", {
throwOnError: true,
});
expect(compiled).toEqual(oracle);
});
});
});
import { lexify, parse, Node } from "metabase/lib/expressions/pratt";
describe("metabase/lib/expressions/parser", () => {
interface AST {
token: string;
children: AST[];
pos: number;
}
function cleanupAST(node: Node): AST {
return {
token: node.token?.text || "UNKNOWN",
children: node.children.map(cleanupAST),
pos: node.token?.pos || -1,
};
}
function parseExpression(source: string, throwOnError: boolean = true) {
return cleanupAST(
parse(lexify(source), {
throwOnError,
}).root,
);
}
const parseAggregation = parseExpression;
const parseFilter = parseExpression;
describe("Handles crazy expressions", () => {
it("Seed 66120", () => {
const expr = `(Z8YZP(_1(- CW(182751, (_d_YU_((h3M_))), 0e31, 0) > _d((0e+96) >= [UVz], NOT ltKvh([__7g], (\"85\")) / - (D___(h627I1 (50106.e80, Qh8_U(([B] = \"d9 \"), 586107.E1 OR [__42 → G_TH]), [Cc_]) + 22882e+22, \"ws\" * 642705.632614e+47, - _2Y_nZi + - __k1_q_(((519470.E+61)), [D]) >= (0E20), [__1sCC3 → mz8])), (M(Q4__L(), (NOT NOT NOT NOT - \"g\" + NOT 0e+35))) <= 798469.324524E54)), h((- R8__K), T6([f3w67], NOT [D8d2933])), ftI7YA(0E+10 <= [F1Y_ → tuj] * 907239. = (NOT O1sW7pq((631184.576387e+12 > _H___3B(- [v_h_4])) != - z) <= ([_EvzL_8 → Ar]) <= [e1_Jy57]) AND _u2_(252719, \" 9\"), __(hB(NOT \"7j 1_V_\", 660445E+96, (x0E9ox(k__p(_9h5_() AND e(\"70_f\", __2_S_Q()) + [__W6U → L7UnD6J] = (NOT NOT N(h_9(\"4 g \", 575068.), NOT \"487\", [S → yWQ_], (\" \" + \" k_ dd2d \"))), (TMIVF(\"c\", - [_gx3sJ → g___5j], (559929.E66))), - [jUc_y → o7], p84__t8((((Mg8_()))), - [ZF59_w8 → U], - [T_ → gw_2k3] OR (848268.635035E+38 * (20814.e-38)), P_7_0 <= y09(C3_(399920), Lw_O(- 252450.180012e-81 = 290101.992467, 71543.409973E65, A((_()), H(858127., 1, \"_W D\", \" __\"), - 253307.244574e21, [IZ_Q_D])), _q(\"_ _\"), rRa2) > Q(_1Ux0, NOT K__j89, - (V_7IS), \"_n58\"))))) - 719454), sq_R(), 346932., NOT B___())), - 787799.)) < \"712\"`;
expect(() => parseExpression(expr)).not.toThrow();
});
it("Seed 635496", () => {
const expr = `K__(m_AA(yyg6_t((NOT [XH8]), [W → PU]) > [k8xNd3 → md_zfIs] + a9_H9MP, a___tf(p_G(- C_Ug((_5iP), (vh_) > _zaS_, 578833.e+0 = ([_9S_64] <= C_(t53I, Q_6(NOT MO_B(0, NOT K61p_f)))))), (866983.)), __E(_W7a_, oXm7A(_01E_(627139.300685 > NOT u_1_d6(p4564j1(gZfe_9((- 727986.), p7hU()) * - - U2w_k9_(0, 1, \"69 a\", \" \"), _129)) <= NOT \"k_K\", - (_chTf5V() AND E3_(u_X2_, (NOT qImu5()), 955004) = ZQcm <= 96160.580572), 904595.), 46320.407574E+99) > [hXw → H_KD], NOT 126223.561027, (C_I92O) OR (NOT Je5_C([UvO_b1 → _i_S], 0, 647472E+81 != - __(), 920107.)) != (NOT f(68493., 335976.415413 > (1))) != _(_y(\"___\", [___u_L], [_w], [ODVlD → nM])) > \"Qq \"), ((\"7\") * 754944.E85)) >= (- XzB__(t1782, (\"r_\") <= \"__wa\") AND - 107898.37690E+16)) = (NOT [J_ → _])`;
expect(() => parseExpression(expr)).not.toThrow();
});
it("Seed 59999", () => {
const expr = `_z * NOT 0 = TS(__O_v2_(_t(- (- Y_), (NOT _27a != NOT T80Bm67((20804.), 664158.926033 AND [B82PH_], - NOT (891819 < \"a_wf_DI\") - 731090. >= 1 <= ((_g0AC(867927E+79)) <= \"_P \" = 990643.), ([yL4P_0 → _])) >= gn7_19(0 AND um6(((((202340.)))), NOT \"P\", (0 + - \"_l4k3\" = (_HG7_B(\"8t5_s _O2\", 1, \"Y_d_\", 0)))), [L2_], NOT 648865.E73, - - (355186e+44)))), [_G], [__Z → G_U9], NOT (NOT 756239.969634E-67 = ((_baX_r())) + ((NOT - [Q4 → pB]))) AND f__ErS(766747, [hc0y → sjA_h], [_ → v7__kJ1] * (D1_n18(_2B(1))) AND NOT u7(vj__) OR 847581, NOT (1e+5))), \"WZztF\")`;
expect(() => parseExpression(expr)).not.toThrow();
});
});
describe("handles fields", () => {
// ---- Negation - number ----
it("should accept a field", () => {
expect(() => parseExpression("[A]")).not.toThrow();
});
it("should accept a negative field", () => {
expect(() => parseExpression("-[A]")).not.toThrow();
});
it("should accept a field with escaped parentheses", () => {
expect(() => parseExpression("[A \\(B\\)]")).not.toThrow();
});
it("should accept an escaped field", () => {
expect(() => parseExpression("[A \\[T\\]]")).not.toThrow();
});
});
describe("handles negation", () => {
// ---- Negation - number ----
it("should accept a negative number", () => {
expect(() => parseExpression("-42")).not.toThrow();
});
it("should accept a negative number (outside parentheses)", () => {
expect(() => parseExpression("-(42)")).not.toThrow();
});
it("should accept a negative number (inside parentheses)", () => {
expect(() => parseExpression("(-42)")).not.toThrow();
});
// ---- Negation - identifier ----
it("should accept a negative identifier", () => {
expect(() => parseExpression("-[X]")).not.toThrow();
});
it("should accept a negative identifier (outside parentheses)", () => {
expect(() => parseExpression("-([X])")).not.toThrow();
});
it("should accept a negative identifier (inside parentheses)", () => {
expect(() => parseExpression("(-[X])")).not.toThrow();
});
// ---- Negation - identifier ----
it("should accept a negative function call", () => {
expect(() => parseExpression("-abs([X])")).not.toThrow();
});
it("should accept a negative function call (outside parentheses)", () => {
expect(() => parseExpression("-(abs([X]))")).not.toThrow();
});
it("should accept a negative function call (inside parentheses)", () => {
expect(() => parseExpression("(-abs([X]))")).not.toThrow();
});
/// ---- Other weird negations ----
it("should accept a double negation with syntax error", () => {
expect(() => parseExpression("NOT NOT Or", false)).not.toThrow();
});
});
describe("(in expression mode)", () => {
it("should accept a number", () => {
expect(() => parseExpression("42")).not.toThrow();
});
it("should accept a single-quoted string", () => {
expect(() => parseExpression("'Answer'")).not.toThrow();
});
it("should accept an escaped single-quoted string", () => {
expect(() => parseExpression("'An\\'swer'")).not.toThrow();
});
it("should accept a double-quoted string", () => {
expect(() => parseExpression('"Answer"')).not.toThrow();
});
it("should accept an escaped double-quoted string", () => {
expect(() => parseExpression('"An\\"swer"')).not.toThrow();
});
it("should accept a group expression (in parentheses)", () => {
expect(() => parseExpression("(42)")).not.toThrow();
});
it("should accept the function lower", () => {
expect(() => parseExpression("Lower([Title])")).not.toThrow();
});
it("should accept the function upper", () => {
expect(() => parseExpression("Upper([Title])")).not.toThrow();
});
it("should accept the function CASE", () => {
expect(() => parseExpression("Case([Z]>7, 'X', 'Y')")).not.toThrow();
});
it("should accept the function CASE with multiple cases", () => {
expect(() => parseExpression("Case([X]>5,5,[X]>3,3,0)")).not.toThrow();
});
it("should reject an unclosed single-quoted string", () => {
expect(() => parseExpression('"Answer')).toThrow();
});
it("should reject an unclosed double-quoted string", () => {
expect(() => parseExpression('"Answer')).toThrow();
});
it("should reject a mismatched quoted string", () => {
expect(() => parseExpression("\"Answer'")).toThrow();
});
it("should handle a conditional with ISEMPTY", () => {
expect(() =>
parseExpression("case(isempty([Discount]),[P])"),
).not.toThrow();
});
// TODO: This should be handled by a separate pass
xit("should reject CASE with only one argument", () => {
expect(() => parseExpression("case([Deal])")).toThrow();
});
it("should accept CASE with two arguments", () => {
expect(() => parseExpression("case([Deal],x)")).not.toThrow();
});
it("should reject CASE missing a closing paren", () => {
expect(() => parseExpression("case([Deal],x")).toThrow();
});
});
describe("(in aggregation mode)", () => {
it("should accept an aggregration with COUNT", () => {
expect(() => parseAggregation("Count()")).not.toThrow();
});
it("should accept an aggregration with SUM", () => {
expect(() => parseAggregation("Sum([Price])")).not.toThrow();
});
it("should accept an aggregration with DISTINCT", () => {
expect(() => parseAggregation("Distinct([Supplier])")).not.toThrow();
});
it("should accept an aggregration with STANDARDDEVIATION", () => {
expect(() => parseAggregation("StandardDeviation([Debt])")).not.toThrow();
});
it("should accept an aggregration with AVERAGE", () => {
expect(() => parseAggregation("Average([Height])")).not.toThrow();
});
it("should accept an aggregration with MAX", () => {
expect(() => parseAggregation("Max([Discount])")).not.toThrow();
});
it("should accept an aggregration with MIN", () => {
expect(() => parseAggregation("Min([Rating])")).not.toThrow();
});
it("should accept an aggregration with MEDIAN", () => {
expect(() => parseAggregation("Median([Total])")).not.toThrow();
});
it("should accept an aggregration with VAR", () => {
expect(() => parseAggregation("Variance([Tax])")).not.toThrow();
});
it("should accept a conditional aggregration with COUNTIF", () => {
expect(() => parseAggregation("CountIf([Discount] > 0)")).not.toThrow();
});
it("should accept a conditional aggregration with COUNTIF containing an expression", () => {
expect(() => parseAggregation("CountIf(([A]+[B]) > 1)")).not.toThrow();
expect(() =>
parseAggregation("CountIf( 1.2 * [Price] > 37)"),
).not.toThrow();
});
});
describe("(in filter mode)", () => {
it("should accept a simple comparison", () => {
expect(() => parseFilter("[Total] > 12")).not.toThrow();
});
it("should accept another simple comparison", () => {
expect(() => parseFilter("10 < [DiscountPercent]")).not.toThrow();
});
it("should accept a logical NOT", () => {
expect(() => parseFilter("NOT [Debt] > 5")).not.toThrow();
});
it("should accept a segment", () => {
expect(() => parseFilter("[SpecialDeal]")).not.toThrow();
});
it("should accept a logical NOT on segment", () => {
expect(() => parseFilter("NOT [Clearance]")).not.toThrow();
});
it("should accept multiple logical NOTs on segment", () => {
expect(() => parseFilter("NOT NOT [Clearance]")).not.toThrow();
});
it("should accept a relational between a segment and a dimension", () => {
expect(() => parseFilter("([Shipping] < 2) AND [Sale]")).not.toThrow();
});
it("should accept parenthesized logical operations", () => {
expect(() => parseFilter("([Deal] AND [HighRating])")).not.toThrow();
expect(() => parseFilter("([Price] < 100 OR [Refurb])")).not.toThrow();
});
it("should accept a function", () => {
expect(() => parseFilter("between([Subtotal], 1, 2)")).not.toThrow();
});
// TODO: This should be handled by a separate pass
xit("should reject CASE with only one argument", () => {
expect(() => parseFilter("case([Deal])")).toThrow();
});
});
});
......@@ -49,7 +49,7 @@ describe("scenarios > question > custom column > expression editor", () => {
.type("{movetoend}{backspace}", { force: true })
.blur();
cy.findByText("Unexpected end of input");
cy.findByText("Expected expression");
cy.button("Done").should("be.disabled");
});
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment