Custom expression editor: use new parser/compiler to generate diagnostics (#19557)

* pratt parser diagnostics

* ported over tests from custom parser branch

* updated error types to support pos/len for error markers

* updated compiler error to removed unnecessary plural

* updated resolver to use new error type and pass through node information

* updated error text in custom column e2e test

* updated diagnostics to use new resolver error class

* update recursive-parser passes to pass through node information

* roll back error text fix in e2e tests

* deleted fuzzing tests (they will be moved into the metabase/expression repo)

* this is weird, setting e2e error text back to "Expected expression"

* rolled back expression editor changes in handleInputBlur and moved diagnoseExpression first to fix exponent rewrite issue

* roll back changes to compileExpression

* update import paths in expressions case class identities get messed up

frontend/src/metabase/lib/expressions/diagnostics.js

 import { t } from "ttag";
 
+import {
+  getMBQLName,
+  parseDimension,
+  parseMetric,
+  parseSegment,
+} from "metabase/lib/expressions";
+import { resolve } from "metabase/lib/expressions/resolver";
+import {
+  parse,
+  lexify,
+  compile,
+  ResolverError,
+} from "metabase/lib/expressions/pratt";
+import {
+  useShorthands,
+  adjustCase,
+  adjustOptions,
+} from "metabase/lib/expressions/recursive-parser";
 import { tokenize, TOKEN, OPERATOR } from "metabase/lib/expressions/tokenizer";
-import { getMBQLName } from "metabase/lib/expressions";
-import { processSource } from "metabase/lib/expressions/process";
 
 // e.g. "COUNTIF(([Total]-[Tax] <5" returns 2 (missing parentheses)
 export function countMatchingParentheses(tokens) {
@@ -53,12 +69,66 @@ export function diagnose(source, startRule, query) {
     return { message };
   }
 
-  const { compileError } = processSource({ source, query, startRule });
+  try {
+    return prattCompiler(source, startRule, query);
+  } catch (err) {
+    return err;
+  }
+}
+
+function prattCompiler(source, startRule, query) {
+  const tokens = lexify(source);
+  const options = { source, startRule, query };
+
+  // PARSE
+  const { root, errors } = parse(tokens, {
+    throwOnError: false,
+    ...options,
+  });
+  if (errors.length > 0) {
+    return errors[0];
+  }
+
+  function resolveMBQLField(kind, name, node) {
+    if (!query) {
+      return [kind, name];
+    }
+    if (kind === "metric") {
+      const metric = parseMetric(name, options);
+      if (!metric) {
+        throw new ResolverError(`Unknown Field: ${name}`, node);
+      }
+      return ["metric", metric.id];
+    } else if (kind === "segment") {
+      const segment = parseSegment(name, options);
+      if (!segment) {
+        throw new ResolverError(`Unknown Field: ${name}`, node);
+      }
+      return ["segment", segment.id];
+    } else {
+      // fallback
+      const dimension = parseDimension(name, options);
+      if (!dimension) {
+        throw new ResolverError(`Unknown Field: ${name}`, node);
+      }
+      return dimension.mbql();
+    }
+  }
 
-  if (compileError) {
-    return Array.isArray(compileError) && compileError.length > 0
-      ? compileError[0]
-      : compileError;
+  // COMPILE
+  try {
+    compile(root, {
+      passes: [
+        adjustOptions,
+        useShorthands,
+        adjustCase,
+        expr => resolve(expr, startRule, resolveMBQLField),
+      ],
+      getMBQLName,
+    });
+  } catch (err) {
+    console.warn("compile error", err);
+    return err;
   }
 
   return null;

frontend/src/metabase/lib/expressions/pratt/compiler.ts

+import {
+  /* ALL_ASTYPES */ ADD,
+  FIELD,
+  LOGICAL_AND,
+  CALL,
+  EQUALITY,
+  NUMBER,
+  LOGICAL_OR,
+  COMPARISON,
+  GROUP,
+  MULDIV_OP,
+  STRING,
+  SUB,
+  NEGATIVE,
+  LOGICAL_NOT,
+  IDENTIFIER,
+  ROOT,
+  ARG_LIST,
+} from "metabase/lib/expressions/pratt/syntax";
+import {
+  assert,
+  NodeType,
+  Node,
+  CompileError,
+} from "metabase/lib/expressions/pratt/types";
+
+export type Expr = number | string | ([string, ...Expr[]] & { node?: Node });
+export type CompilerPass = (expr: Expr) => Expr;
+
+export interface Options {
+  getMBQLName(expressionName: string): string | undefined;
+  passes?: CompilerPass[];
+}
+
+type CompileFn = (node: Node, opts: Options) => Expr;
+
+export function compile(node: Node, opts: Options): Expr {
+  assert(node.type === ROOT, "Must be root node");
+  if (node.children.length > 1) {
+    throw new CompileError("Unexpected expression", {
+      node: node.children[1],
+      token: node.children[1].token,
+    });
+  }
+  const func = compileUnaryOp(node);
+  let expr = func(node.children[0], opts);
+  const { passes = [] } = opts;
+  for (const pass of passes) {
+    expr = pass(expr);
+  }
+  return expr;
+}
+
+// ----------------------------------------------------------------
+
+function compileField(node: Node): Expr {
+  assert(node.type === FIELD, "Invalid Node Type");
+  assert(node.token?.text, "Empty field name");
+  // Slice off the leading and trailing brackets
+  const name = node.token.text.slice(1, node.token.text.length - 1);
+  return withNode(["dimension", name], node);
+}
+
+function compileIdentifier(node: Node): Expr {
+  assert(node.type === IDENTIFIER, "Invalid Node Type");
+  assert(node.token?.text, "Empty token text");
+  const name = node.token.text;
+  return withNode(["dimension", name], node);
+}
+
+function compileGroup(node: Node, opts: Options): Expr {
+  assert(node.type === GROUP, "Invalid Node Type");
+  const func = compileUnaryOp(node);
+  return func(node.children[0], opts);
+}
+
+function compileString(node: Node): Expr {
+  assert(node.type === STRING, "Invalid Node Type");
+  assert(typeof node.token?.text === "string", "No token text");
+  // Slice off the leading and trailing quotes
+  return node.token.text.slice(1, node.token.text.length - 1);
+}
+
+// ----------------------------------------------------------------
+
+function compileLogicalNot(node: Node, opts: Options): Expr {
+  assert(node.type === LOGICAL_NOT, "Invalid Node Type");
+  const func = compileUnaryOp(node);
+  assert(node.token?.text, "Empty token text");
+  const child = node.children[0];
+  return withNode(["not", func(child, opts)], node);
+}
+
+function compileLogicalAnd(node: Node, opts: Options): Expr {
+  assert(node.type === LOGICAL_AND, "Invalid Node Type");
+  assert(node.token?.text, "Empty token text");
+  const [left, right] = compileInfixOp(node, opts);
+  return withNode([node.token?.text.toLowerCase(), ...left, ...right], node);
+}
+
+function compileLogicalOr(node: Node, opts: Options): Expr {
+  assert(node.type === LOGICAL_OR, "Invalid Node Type");
+  assert(node.token?.text, "Empty token text");
+  const [left, right] = compileInfixOp(node, opts);
+  return withNode([node.token?.text.toLowerCase(), ...left, ...right], node);
+}
+
+function compileComparisonOp(node: Node, opts: Options): Expr {
+  assert(node.type === COMPARISON, "Invalid Node Type");
+  const text = node.token?.text;
+  assert(text, "Empty token text");
+  const [left, right] = compileInfixOp(node, opts);
+  return withNode([text, ...left, ...right], node);
+}
+
+function compileEqualityOp(node: Node, opts: Options): Expr {
+  assert(node.type === EQUALITY, "Invalid Node Type");
+  assert(node.token?.text, "Empty token text");
+  const [left, right] = compileInfixOp(node, opts);
+  return withNode([node.token?.text, ...left, ...right], node);
+}
+
+// ----------------------------------------------------------------
+
+function compileFunctionCall(node: Node, opts: Options): Expr {
+  assert(node.type === CALL, "Invalid Node Type");
+  assert(node.token?.text, "Empty token text");
+  assert(
+    node.children[0].type === ARG_LIST,
+    "First argument must be an arglist",
+  );
+  const text = node.token?.text;
+  const fn = opts.getMBQLName(text.trim().toLowerCase());
+  return withNode(
+    [fn ? fn : text, ...compileArgList(node.children[0], opts)],
+    node,
+  );
+}
+
+function compileArgList(node: Node, opts: Options): Expr[] {
+  assert(node.type === ARG_LIST, "Invalid Node Type");
+  return node.children.map(child => {
+    const func = getCompileFunction(child);
+    if (!func) {
+      throw new CompileError("Invalid node type", { node: child });
+    }
+    const expr = func(child, opts);
+    return (expr as any).node ? expr : withNode(expr, child);
+  });
+}
+
+// ----------------------------------------------------------------
+
+function compileNumber(node: Node): Expr {
+  assert(node.type === NUMBER, "Invalid Node Type");
+  assert(typeof node.token?.text === "string", "No token text");
+  try {
+    return parseFloat(node.token.text);
+  } catch (err) {
+    throw new CompileError("Invalid number format", {
+      node,
+      token: node.token,
+    });
+  }
+}
+
+function compileNegative(node: Node, opts: Options): Expr {
+  assert(node.type === NEGATIVE, "Invalid Node Type");
+  const func = compileUnaryOp(node);
+  assert(node.token?.text, "Empty token text");
+  const child = node.children[0];
+  if (child.type === NUMBER) {
+    return -func(child, opts);
+  }
+  return withNode(["-", func(child, opts)], node);
+}
+
+function compileAdditionOp(node: Node, opts: Options): Expr {
+  assert(node.type === ADD, "Invalid Node Type");
+  assert(node.token?.text, "Empty token text");
+  const [left, right] = compileInfixOp(node, opts);
+  return withNode([node.token?.text, ...left, ...right], node);
+}
+
+function compileMulDivOp(node: Node, opts: Options): Expr {
+  assert(node.type === MULDIV_OP, "Invalid Node Type");
+  const text = node.token?.text;
+  assert(text, "Empty token text");
+  const [left, right] = compileInfixOp(node, opts);
+  return withNode([text, ...left, ...right], node);
+}
+
+function compileSubtractionOp(node: Node, opts: Options): Expr {
+  assert(node.type === SUB, "Invalid Node Type");
+  assert(node.token?.text, "Empty token text");
+  const [left, right] = compileInfixOp(node, opts);
+  return withNode([node.token?.text, ...left, ...right], node);
+}
+
+// ----------------------------------------------------------------
+
+function compileUnaryOp(node: Node) {
+  if (node.children.length > 1) {
+    throw new CompileError("Unexpected expression", {
+      node: node.children[1],
+      token: node.children[1].token,
+    });
+  } else if (node.children.length === 0) {
+    throw new CompileError("Expected expression", { node, token: node.token });
+  }
+  const func = getCompileFunction(node.children[0]);
+  if (!func) {
+    throw new CompileError("Invalid node type", {
+      node: node.children[0],
+      token: node.children[0].token,
+    });
+  }
+  return func;
+}
+
+function compileInfixOp(node: Node, opts: Options) {
+  if (node.children.length > 2) {
+    throw new CompileError("Unexpected expression", {
+      node: node.children[2],
+      token: node.children[2].token,
+    });
+  } else if (node.children.length === 0) {
+    throw new CompileError("Expected expression", { node, token: node.token });
+  }
+  const leftFn = getCompileFunction(node.children[0]);
+  if (!leftFn) {
+    throw new CompileError("Invalid node type", {
+      node: node.children[0],
+      token: node.children[0].token,
+    });
+  }
+  const rightFn = getCompileFunction(node.children[1]);
+  if (!rightFn) {
+    throw new CompileError("Invalid node type", {
+      node: node.children[1],
+      token: node.children[1].token,
+    });
+  }
+
+  const text = node.token?.text;
+  let left: any = leftFn(node.children[0], opts);
+  if (Array.isArray(left) && left[0]?.toUpperCase() === text?.toUpperCase()) {
+    const [_, ...args] = left;
+    left = args;
+  } else {
+    left = [left];
+  }
+
+  let right: any = rightFn(node.children[1], opts);
+  right = [right];
+  return [left, right];
+}
+
+function withNode<T>(expr: T, node: Node): T {
+  if (typeof expr === "object") {
+    Object.defineProperty(expr, "node", {
+      writable: false,
+      enumerable: false,
+      value: node,
+    });
+  }
+  return expr;
+}
+
+// ----------------------------------------------------------------
+
+const COMPILE = new Map<NodeType, CompileFn>([
+  [FIELD, compileField],
+  [ADD, compileAdditionOp],
+  [LOGICAL_AND, compileLogicalAnd],
+  [CALL, compileFunctionCall],
+  [EQUALITY, compileEqualityOp],
+  [NUMBER, compileNumber],
+  [LOGICAL_NOT, compileLogicalNot],
+  [NEGATIVE, compileNegative],
+  [LOGICAL_OR, compileLogicalOr],
+  [COMPARISON, compileComparisonOp],
+  [GROUP, compileGroup],
+  [MULDIV_OP, compileMulDivOp],
+  [STRING, compileString],
+  [SUB, compileSubtractionOp],
+  [IDENTIFIER, compileIdentifier],
+]);
+
+function getCompileFunction(node: Node): (node: Node, opts: Options) => Expr {
+  const func = COMPILE.get(node.type);
+  if (!func) {
+    throw new CompileError("Invalid node type", { node, token: node.token });
+  }
+  return func;
+}

frontend/src/metabase/lib/expressions/pratt/index.ts

+export * from "metabase/lib/expressions/pratt/parser";
+export * from "metabase/lib/expressions/pratt/compiler";
+export * from "metabase/lib/expressions/pratt/syntax";
+export * from "metabase/lib/expressions/pratt/types";

frontend/src/metabase/lib/expressions/pratt/parser.ts

+import {
+  ADD,
+  ARG_LIST,
+  BAD_TOKEN,
+  CALL,
+  COMMA,
+  COMPARISON,
+  END_OF_INPUT,
+  EQUALITY,
+  FIELD,
+  GROUP,
+  GROUP_CLOSE,
+  IDENTIFIER,
+  LOGICAL_AND,
+  LOGICAL_NOT,
+  LOGICAL_OR,
+  MULDIV_OP,
+  NEGATIVE,
+  NUMBER,
+  ROOT,
+  STRING,
+  SUB,
+  WS,
+} from "metabase/lib/expressions/pratt/syntax";
+import {
+  assert,
+  CompileError,
+  NodeType,
+  Token,
+  Node,
+  Hooks,
+} from "metabase/lib/expressions/pratt/types";
+
+import { tokenize, TOKEN, OPERATOR } from "metabase/lib/expressions/tokenizer";
+
+interface ParserOptions {
+  hooks?: Hooks;
+  maxIterations?: number;
+  throwOnError?: boolean;
+}
+
+interface ParserResult {
+  root: Node;
+  errors: CompileError[];
+}
+
+export function lexify(expression: string) {
+  const lexs: Token[] = [];
+
+  const { tokens, errors } = tokenize(expression);
+  if (errors && errors.length > 0) {
+    errors.forEach(error => {
+      const { pos } = error;
+      lexs.push({ type: BAD_TOKEN, text: expression[pos], length: 1, pos });
+    });
+  }
+
+  let start = 0;
+  for (let i = 0; i < tokens.length; ++i) {
+    const token = tokens[i];
+    if (start < token.start) {
+      lexs.push({
+        type: WS,
+        text: expression.slice(start, token.start),
+        length: token.start - start,
+        pos: start,
+      });
+    }
+    start = token.end;
+    let text = expression.slice(token.start, token.end);
+    const pos = token.start;
+    let length = token.end - token.start;
+    let type = BAD_TOKEN;
+    switch (token.type) {
+      case TOKEN.Number:
+        type = NUMBER;
+        break;
+      case TOKEN.String:
+        type = STRING;
+        break;
+      case TOKEN.Identifier:
+        type = text[0] === "[" ? FIELD : IDENTIFIER;
+        break;
+      case TOKEN.Operator:
+        switch (token.op) {
+          case OPERATOR.Comma:
+            type = COMMA;
+            break;
+          case OPERATOR.OpenParenthesis:
+            type = GROUP;
+            break;
+          case OPERATOR.CloseParenthesis:
+            type = GROUP_CLOSE;
+            break;
+          case OPERATOR.Plus:
+            type = ADD;
+            break;
+          case OPERATOR.Minus:
+            type = SUB;
+            break;
+          case OPERATOR.Star:
+          case OPERATOR.Slash:
+            type = MULDIV_OP;
+            break;
+          case OPERATOR.Equal:
+          case OPERATOR.NotEqual:
+            type = EQUALITY;
+            break;
+          case OPERATOR.LessThan:
+          case OPERATOR.GreaterThan:
+          case OPERATOR.LessThanEqual:
+          case OPERATOR.GreaterThanEqual:
+            type = COMPARISON;
+            break;
+          case OPERATOR.Not:
+            type = LOGICAL_NOT;
+            break;
+          case OPERATOR.And:
+            type = LOGICAL_AND;
+            break;
+          case OPERATOR.Or:
+            type = LOGICAL_OR;
+            break;
+          default:
+            break;
+        }
+        break;
+    }
+
+    if (type === IDENTIFIER) {
+      const next = tokens[i + 1];
+      if (
+        next &&
+        next.type === TOKEN.Operator &&
+        next.op === OPERATOR.OpenParenthesis
+      ) {
+        type = CALL;
+        length = next.start - token.start;
+        text = expression.slice(token.start, next.start);
+        start = next.start;
+      }
+    }
+
+    lexs.push({ type, text, length, pos });
+  }
+
+  // This simplifies the parser
+  lexs.push({
+    type: END_OF_INPUT,
+    text: "\n",
+    length: 1,
+    pos: expression.length,
+  });
+
+  return lexs.sort((a, b) => a.pos - b.pos);
+}
+
+export function parse(tokens: Token[], opts: ParserOptions = {}): ParserResult {
+  const { maxIterations = 1000000, hooks = {}, throwOnError = false } = opts;
+  const errors: CompileError[] = [];
+  let counter = 0;
+  const root = createASTNode(null, null, ROOT, counter);
+  root.isRoot = true;
+
+  let node = root;
+  hooks.onCreateNode?.(tokens[0], node);
+  for (
+    let index = 0;
+    index < tokens.length && counter < maxIterations;
+    index++
+  ) {
+    const token = tokens[index];
+    hooks.onIteration?.(token, node);
+
+    if (token.type.skip) {
+      hooks.onSkipToken?.(token, node);
+      continue;
+    }
+    if (token.type === BAD_TOKEN) {
+      const err = new CompileError(`Unexpected token "${token.text}"`, {
+        node,
+        token,
+      });
+      hooks.onBadToken?.(token, node, err);
+      if (throwOnError) {
+        throw err;
+      }
+      errors.push(err);
+      // If we don't throw on error, we skip the bad token
+      continue;
+    }
+
+    if (node.complete) {
+      // If a node has received all the children it expects, it's time to figure
+      // out whether it needs to be reparented. This is the core of the
+      // our solution to the predence issue. By default, we can expect the node
+      // to go to its parent but if the next token has a higher precedence (like
+      // `*` over `+`), it might take the node instead.
+      assert(
+        node.parent,
+        "Marked a node complete without placing it with a parent",
+      );
+
+      // This is the core of the precedence climbing logic. If a higher priority
+      // token is encountered, shouldReparent will return true and the new node
+      // we created for the token will "take" the current node
+      if (shouldReparent(node.parent.type, token.type)) {
+        node.parent = createASTNode(
+          token,
+          node.parent,
+          getASType(token.type, node.parent.type),
+          counter,
+        );
+        hooks.onReparentNode?.(token, node);
+      } else {
+        // If we don't need to reparent, we decrement the token index. This is
+        // because we iterate several times for each node, first to create it
+        // and then to check if it is completed.
+        index--;
+      }
+
+      // Place the node in its parent children and get the next "active" node
+      // which is node.parent
+      node = place(node, errors, opts);
+      if (node.children.length === node.type.expectedChildCount) {
+        node.complete = true;
+        hooks.onCompleteNode?.(token, node);
+      }
+    } else if (token.type.isTerminator) {
+      hooks.onTerminatorToken?.(token, node);
+      // Terminator tokens like `]`, `)` or end of input will complete a node if
+      // they match the type's `requiresTerminator`
+      if (node.type.requiresTerminator === token.type) {
+        node.complete = true;
+        hooks.onCompleteNode?.(token, node);
+      } else if (node.type.ignoresTerminator.indexOf(token.type) === -1) {
+        // If the current token isn't in the list of the AST type's ignored
+        // tokens and it's not the terminator the current node requires, we'll
+        // throw an error
+        const err = new CompileError(`Expected expression`, { node, token });
+        hooks.onUnexpectedTerminator?.(token, node, err);
+        if (throwOnError) {
+          throw err;
+        }
+        errors.push(err);
+
+        if (token.type === END_OF_INPUT) {
+          // We complete and reparent/place the final node by running the for
+          // loop one last time
+          if (!node.complete) {
+            node.complete = true;
+            hooks.onCompleteNode?.(token, node);
+            index--;
+          }
+        }
+      }
+    } else if (token.type.leftOperands !== 0) {
+      // Subtraction is a special case because it might actually be negation
+      if (token.type === SUB) {
+        node = createASTNode(token, node, NEGATIVE, counter);
+        hooks.onCreateNode?.(token, node);
+      } else {
+        const err = new CompileError(`Expected expression`, {
+          token,
+        });
+        hooks.onMissinChildren?.(token, node, err);
+        if (throwOnError) {
+          throw err;
+        }
+        errors.push(err);
+      }
+    } else {
+      // Create the AST node. It will be marked as complete if the node doesn't
+      // expect any children (like a literal or identifier)
+      node = createASTNode(
+        token,
+        node,
+        getASType(token.type, node.type),
+        counter,
+      );
+      hooks.onCreateNode?.(token, node);
+    }
+    counter += 1;
+  }
+
+  if (counter >= maxIterations) {
+    throw new Error("Reached max number of iterations");
+  }
+
+  const childViolation = ROOT.checkChildConstraints(root);
+  if (childViolation !== null) {
+    const err = new CompileError("Unexpected token", {
+      node: root,
+      ...childViolation,
+    });
+    hooks.onChildConstraintViolation?.(node, err);
+    if (throwOnError) {
+      throw err;
+    }
+    errors.push(err);
+  }
+  return { root, errors };
+}
+
+function createASTNode(
+  token: Token | null,
+  parent: Node | null,
+  type: NodeType,
+  counter: number,
+): Node {
+  return {
+    type,
+    children: [],
+    complete: type.expectedChildCount === 0,
+    parent,
+    token,
+    resolvedType: type.resolvesAs ? type.resolvesAs : counter,
+  };
+}
+
+function place(node: Node, errors: CompileError[], opts: ParserOptions) {
+  const { hooks = {}, throwOnError = false } = opts;
+  const { type, parent } = node;
+
+  const childViolation = type.checkChildConstraints(node);
+  if (childViolation !== null) {
+    const err = new CompileError("Unexpected token", {
+      node,
+      ...childViolation,
+    });
+    hooks.onChildConstraintViolation?.(node, err);
+    if (throwOnError) {
+      throw err;
+    }
+    errors.push(err);
+  }
+  assert(parent, "Tried to place a node without a parent", node);
+  parent.children.push(node);
+  hooks.onPlaceNode?.(node, parent);
+  return parent;
+}
+
+function shouldReparent(leftType: NodeType, rightType: NodeType) {
+  // If the right node doesn't have any left operands like a literal or
+  // identifier, then it can't become the parent of the left node anyway
+  if (rightType.leftOperands === 0) {
+    return false;
+  } else {
+    return rightType.precedence > leftType.precedence;
+  }
+}
+
+export function getASType(type: NodeType, parentType: NodeType) {
+  if (type === GROUP) {
+    // A list of function call arguments is first interpreted as a GROUP, then
+    // reinterpreted as an ARG_LIST if its the child of a CALL
+    if (parentType === CALL) {
+      return ARG_LIST;
+    }
+  }
+  return type;
+}

frontend/src/metabase/lib/expressions/pratt/syntax.ts

+import { assert, NodeType, Node } from "metabase/lib/expressions/pratt/types";
+
+/*
+ * This file specifies most of the syntax for the Metabase handwritten custom
+ * expression parser. The rest is contained in the parser special cases
+ *
+ * The structure of this file:
+ *   1. Declare all of the ASTypes (types of nodes that appear in the abstract
+ *       syntax tree)
+ *   2. Associate various properties with those ASTypes that determine the
+ *       overall shape of the grammar (# of operands, scope info, etc.)
+ *   3. Declare more grammatical rules as functions/generate them from tables
+ *       (parent/child Type constraints, operator precedence tiers)
+ *   4. Export getASType(), which the parser uses to override the Type guesses
+ *       that the lexer makes based on contextual information
+ */
+
+export const FIELD = {} as NodeType;
+export const ADD = {} as NodeType;
+export const LOGICAL_AND = {} as NodeType;
+export const ARG_LIST = {} as NodeType;
+export const BAD_TOKEN = {} as NodeType;
+export const CALL = {} as NodeType;
+export const COMMA = {} as NodeType;
+export const END_OF_INPUT = {} as NodeType;
+export const EQUALITY = {} as NodeType;
+export const NUMBER = {} as NodeType;
+export const LOGICAL_NOT = {} as NodeType;
+export const NEGATIVE = {} as NodeType;
+export const LOGICAL_OR = {} as NodeType;
+export const COMPARISON = {} as NodeType;
+export const GROUP = {} as NodeType;
+export const GROUP_CLOSE = {} as NodeType;
+export const ROOT = {} as NodeType;
+export const MULDIV_OP = {} as NodeType;
+export const STRING = {} as NodeType;
+export const SUB = {} as NodeType;
+export const IDENTIFIER = {} as NodeType;
+export const WS = {} as NodeType;
+
+function operand(leftOperands: number, rightOperands: number) {
+  return {
+    leftOperands,
+    rightOperands,
+    expectedChildCount: leftOperands + rightOperands,
+  };
+}
+
+function setAttributes(...syntaxRules: [Partial<NodeType>, NodeType[]][]) {
+  for (const [values, types] of syntaxRules) {
+    for (const type of types) {
+      Object.assign(type, values);
+    }
+  }
+}
+
+const ALL_NODES = [
+  ADD,
+  LOGICAL_AND,
+  ARG_LIST,
+  BAD_TOKEN,
+  CALL,
+  COMMA,
+  END_OF_INPUT,
+  EQUALITY,
+  NUMBER,
+  NEGATIVE,
+  LOGICAL_NOT,
+  LOGICAL_OR,
+  COMPARISON,
+  GROUP,
+  GROUP_CLOSE,
+  ROOT,
+  MULDIV_OP,
+  STRING,
+  SUB,
+  FIELD,
+  IDENTIFIER,
+  WS,
+];
+
+// Set default values for AST node attributes
+setAttributes([
+  {
+    skip: false,
+
+    leftOperands: 0,
+    rightOperands: 0,
+    expectedChildCount: 0,
+    checkChildConstraints: () => null,
+
+    requiresTerminator: null,
+    ignoresTerminator: [],
+    isTerminator: false,
+
+    precedence: -Infinity,
+    resolvesAs: null,
+    expectedTypes: null,
+  },
+
+  ALL_NODES,
+]);
+
+setAttributes(
+  // Prefix Operators
+  [operand(0, 1), [CALL, NEGATIVE, LOGICAL_NOT]],
+
+  // Infix Operators
+  [
+    operand(1, 1),
+    [MULDIV_OP, ADD, SUB, COMPARISON, EQUALITY, LOGICAL_AND, LOGICAL_OR],
+  ],
+
+  // Open Expressions (various paren types, blocks, etc.) and their terminators
+  [{ expectedChildCount: Infinity }, [ARG_LIST, ROOT, GROUP]],
+  [{ ignoresTerminator: [COMMA] }, [ARG_LIST]],
+  [{ requiresTerminator: END_OF_INPUT }, [ROOT]],
+  [{ requiresTerminator: GROUP_CLOSE }, [ARG_LIST, GROUP]],
+  [{ isTerminator: true }, [COMMA, END_OF_INPUT, GROUP_CLOSE]],
+
+  // Skip whitespace
+  [{ skip: true }, [WS]],
+
+  // Known types
+  [{ resolvesAs: "string" }, [STRING]],
+  [{ resolvesAs: "number" }, [ADD, NUMBER, NEGATIVE, MULDIV_OP, SUB]],
+  [
+    { resolvesAs: "boolean" },
+    [LOGICAL_AND, EQUALITY, LOGICAL_NOT, LOGICAL_OR, COMPARISON],
+  ],
+
+  // Expected types
+  [
+    { expectedTypes: ["number"] },
+    [ADD, NUMBER, NEGATIVE, MULDIV_OP, SUB, COMPARISON],
+  ],
+  [{ expectedTypes: ["boolean"] }, [LOGICAL_NOT, LOGICAL_AND, LOGICAL_OR]],
+  [{ expectedTypes: ["boolean", "number", "string"] }, [EQUALITY]],
+);
+
+/*
+ * Child constraints govern how many children an AST node can have and where
+ * thare placed relative to the node.
+ *
+ * These are syntax rules, rather than semantic ones, since that is handled
+ * later by a different pass. i.e. LOGICAL_AND and LOGICAL_OR rules don't check
+ * that the right and left side resolve to boolean types.
+ *
+ * `checkChildConstraints` returns an object with diagnostic information if
+ * there is a constraint violation, null otherwise.
+ */
+
+function childConstraintByPosition(...positions: NodeType[][]) {
+  return (node: Node) => {
+    for (let i = 0; i < positions.length; i++) {
+      if (!node.children[i]) {
+        return { position: i, expected: positions };
+      } else if (!positions[i].includes(node.children[i].type)) {
+        return { position: i, child: node.children[i], expected: positions };
+      }
+    }
+    return null;
+  };
+}
+
+LOGICAL_NOT.checkChildConstraints = childConstraintByPosition([
+  FIELD,
+  IDENTIFIER,
+  LOGICAL_NOT,
+  LOGICAL_OR,
+  LOGICAL_AND,
+  COMPARISON,
+  EQUALITY,
+  CALL,
+  GROUP,
+
+  NEGATIVE,
+  NUMBER,
+  STRING,
+  ADD,
+  SUB,
+  MULDIV_OP,
+]);
+
+NEGATIVE.checkChildConstraints = childConstraintByPosition([
+  NUMBER,
+  FIELD,
+  IDENTIFIER,
+  NEGATIVE,
+  CALL,
+  GROUP,
+  ADD,
+  SUB,
+  MULDIV_OP,
+
+  LOGICAL_NOT,
+  LOGICAL_OR,
+  LOGICAL_AND,
+  COMPARISON,
+  STRING,
+]);
+CALL.checkChildConstraints = childConstraintByPosition([ARG_LIST]);
+
+function anyChildConstraint(...acceptableTypes: NodeType[]) {
+  return (node: Node) => {
+    for (const child of node.children) {
+      if (!acceptableTypes.includes(child.type)) {
+        return { child };
+      }
+    }
+    return null;
+  };
+}
+
+ROOT.checkChildConstraints = anyChildConstraint(
+  FIELD,
+  ADD,
+  LOGICAL_AND,
+  CALL,
+  EQUALITY,
+  NUMBER,
+  NEGATIVE,
+  LOGICAL_NOT,
+  MULDIV_OP,
+  LOGICAL_OR,
+  COMPARISON,
+  GROUP,
+  STRING,
+  SUB,
+  IDENTIFIER,
+);
+
+/*
+ * This defines the operator precedence in order from highest priority to lowest
+ * priority. When a node with a higher precedence is encountered, the node with
+ * the lower precedence is "reparented" into the higher node.
+ */
+[
+  [CALL],
+  [FIELD],
+  [NEGATIVE],
+  [MULDIV_OP],
+  [ADD, SUB],
+  [EQUALITY, COMPARISON],
+  [LOGICAL_NOT],
+  [LOGICAL_AND],
+  [LOGICAL_OR],
+  [IDENTIFIER],
+].forEach((tier, precedence, tiers) => {
+  for (const type of tier) {
+    type.precedence = tiers.length - precedence;
+  }
+});
+
+// Give each node
+for (const [key, value] of Object.entries(ALL_NODES)) {
+  value.name = key;
+}
+
+SUB.name = "SUBTRACT";
+WS.name = "WHITESPACE";

frontend/src/metabase/lib/expressions/pratt/types.ts

+import { isProduction } from "metabase/env";
+
+export const VOID = Symbol("Unknown Type");
+
+export type VariableKind =
+  | "dimension"
+  | "segment"
+  | "aggregation"
+  | "expression";
+export type Type = VariableKind | "string" | "number" | "boolean";
+export type VariableId = number;
+
+export interface Token {
+  type: NodeType;
+  text: string;
+  length: number;
+  pos: number;
+}
+
+export interface Node {
+  type: NodeType;
+  children: Node[];
+  complete: boolean;
+  resolvedType: Type | VariableId;
+  parent: Node | null;
+  token: Token | null;
+  isRoot?: boolean;
+}
+
+export interface NodeType {
+  name?: string;
+
+  // Should the parser ignore this sort of token entirely (whitespace)
+  skip: boolean;
+
+  // Number of operands to expect for this node on the left side
+  leftOperands: number;
+  // Number of operands to expect for this node on the right side
+  rightOperands: number;
+  // Maximum number of children before this node is considered complete. May be
+  // `Infinity` for nodes lik ARG_LIST, or number of left+right operands
+  expectedChildCount: number;
+  // Check child constraints
+  checkChildConstraints: (
+    node: Node,
+  ) => { position?: number; child?: Node } | null;
+
+  // For open expressions, this is the AST type of tokens that close the
+  // expression  (e.g. GROUP_CLOSE for GROUP).
+  requiresTerminator: NodeType | null;
+  // For open expressions, this is a list of AST types that should be considered
+  // a  "separator" (e.g. COMMA for ARG_LIST).
+  ignoresTerminator: NodeType[];
+  // Does this token type terminate the current expression (unless exempted by
+  // .ignoresTerminator)?
+  isTerminator: boolean;
+
+  // The precedence to use for operator parsing conflicts. Higher wins.
+  precedence: number;
+
+  // The type this node resolves to, if it can be deduced early on. If null, the
+  // parser assigns an integer value for substitutions instead
+  resolvesAs: Type | null;
+
+  // The expectedType of the child nodes
+  expectedTypes: Type[] | null;
+}
+
+type HookFn = (token: Token, node: Node) => void;
+type HookErrFn = (token: Token, node: Node, err: CompileError) => void;
+type NodeErrFn = (node: Node, err: CompileError) => void;
+export interface Hooks {
+  onIteration?: HookFn;
+  onCreateNode?: HookFn;
+  onPlaceNode?: (node: Node, parent: Node) => void;
+  onSkipToken?: HookFn;
+  onReparentNode?: HookFn;
+  onCompleteNode?: HookFn;
+  onTerminatorToken?: HookFn;
+  onBadToken?: HookErrFn;
+  onUnexpectedTerminator?: HookErrFn;
+  onMissinChildren?: HookErrFn;
+  onChildConstraintViolation?: NodeErrFn;
+}
+
+/*
+ * This class helps anything that handles parser errors to use instanceof to
+ * easily distinguish between compilation error exceptions and exceptions due to
+ * bugs
+ */
+export abstract class ExpressionError extends Error {
+  abstract get pos(): number | null;
+  abstract get len(): number | null;
+}
+
+export class CompileError extends ExpressionError {
+  constructor(message: string, private data: any) {
+    super(message);
+  }
+
+  get pos(): number | null {
+    return this.data?.token?.pos ?? null;
+  }
+
+  get len(): number | null {
+    return this.data?.token?.len ?? null;
+  }
+}
+
+export class ResolverError extends ExpressionError {
+  constructor(message: string, private node: Node) {
+    super(message);
+  }
+
+  get pos(): number | null {
+    return this.node?.token?.pos ?? null;
+  }
+
+  get len(): number | null {
+    return this.node?.token?.length ?? null;
+  }
+}
+
+export class AssertionError extends Error {
+  data?: any;
+
+  constructor(message: string, data?: any) {
+    super(`Assertion failed: ${message}`);
+    this.data = data;
+  }
+}
+
+export function assert(
+  condition: any,
+  msg: string,
+  data?: any,
+): asserts condition {
+  if (isProduction) {
+    if (!condition) {
+      throw new AssertionError(msg, data || {});
+    }
+  }
+}

frontend/src/metabase/lib/expressions/recursive-parser.js

@@ -210,12 +210,25 @@ function recursiveParse(source) {
 const modify = (node, transform) => {
   if (Array.isArray(node)) {
     const [operator, ...operands] = node;
-    return transform([
-      operator,
-      ...operands.map(sub => modify(sub, transform)),
-    ]);
+    return withAST(
+      transform([operator, ...operands.map(sub => modify(sub, transform))]),
+      node,
+    );
   }
-  return transform(node);
+  return withAST(transform(node), node);
+};
+
+const withAST = (result, expr) => {
+  // If this expression comes from the compiler, an object property
+  // containing the parent AST node will be included for errors
+  if (expr.node && typeof result.node === "undefined") {
+    Object.defineProperty(result, "node", {
+      writable: false,
+      enumerable: false,
+      value: expr.node,
+    });
+  }
+  return result;
 };
 
 const NEGATIVE_FILTER_SHORTHANDS = {
@@ -233,7 +246,7 @@ export const useShorthands = tree =>
         const [fn, ...params] = operand;
         const shorthand = NEGATIVE_FILTER_SHORTHANDS[fn];
         if (shorthand) {
-          return [shorthand, ...params];
+          return withAST([shorthand, ...params], node);
         }
       }
     }
@@ -259,7 +272,7 @@ export const adjustOptions = tree =>
               operands.pop();
               operands.push({ "include-current": true });
             }
-            return [operator, ...operands];
+            return withAST([operator, ...operands], node);
           }
         }
       }
@@ -282,9 +295,9 @@ export const adjustCase = tree =>
         }
         if (operands.length > 2 * pairCount) {
           const defVal = operands[operands.length - 1];
-          return [operator, pairs, { default: defVal }];
+          return withAST([operator, pairs, { default: defVal }], node);
         }
-        return [operator, pairs];
+        return withAST([operator, pairs], node);
       }
     }
     return node;

frontend/src/metabase/lib/expressions/resolver.js

 import { ngettext, msgid, t } from "ttag";
 
-import { OPERATOR as OP } from "./tokenizer";
-import { MBQL_CLAUSES } from "./index";
+import { OPERATOR as OP } from "metabase/lib/expressions/tokenizer";
+import { ResolverError } from "metabase/lib/expressions/pratt/types";
+import { MBQL_CLAUSES } from "metabase/lib/expressions";
 
 const FIELD_MARKERS = ["dimension", "segment", "metric"];
 const LOGICAL_OPS = [OP.Not, OP.And, OP.Or];
@@ -57,7 +58,7 @@ export function resolve(expression, type = "expression", fn = undefined) {
     if (FIELD_MARKERS.includes(op)) {
       const kind = MAP_TYPE[type] || "dimension";
       const [name] = operands;
-      return fn ? fn(kind, name) : [kind, name];
+      return fn ? fn(kind, name, expression.node) : [kind, name];
     }
 
     let operandType = null;
@@ -69,7 +70,10 @@ export function resolve(expression, type = "expression", fn = undefined) {
       operandType = "expression";
       const [firstOperand] = operands;
       if (typeof firstOperand === "number" && !Array.isArray(firstOperand)) {
-        throw new Error(t`Expecting field but found ${firstOperand}`);
+        throw new ResolverError(
+          t`Expecting field but found ${firstOperand}`,
+          expression.node,
+        );
       }
     } else if (op === "concat") {
       operandType = "expression";
@@ -78,7 +82,10 @@ export function resolve(expression, type = "expression", fn = undefined) {
     } else if (op === "case") {
       const [pairs, options] = operands;
       if (pairs.length < 1) {
-        throw new Error(t`CASE expects 2 arguments or more`);
+        throw new ResolverError(
+          t`CASE expects 2 arguments or more`,
+          expression.node,
+        );
       }
 
       const resolvedPairs = pairs.map(([tst, val]) => [
@@ -105,12 +112,13 @@ export function resolve(expression, type = "expression", fn = undefined) {
 
     const clause = findMBQL(op);
     if (!clause) {
-      throw new Error(t`Unknown function ${op}`);
+      throw new ResolverError(t`Unknown function ${op}`, expression.node);
     }
     const { displayName, args, multiple, hasOptions } = clause;
     if (!isCompatible(type, clause.type)) {
-      throw new Error(
+      throw new ResolverError(
         t`Expecting ${type} but found function ${displayName} returning ${clause.type}`,
+        expression.node,
       );
     }
     if (!multiple) {
@@ -122,12 +130,13 @@ export function resolve(expression, type = "expression", fn = undefined) {
         operands.length < expectedArgsLength ||
         operands.length > maxArgCount
       ) {
-        throw new Error(
+        throw new ResolverError(
           ngettext(
             msgid`Function ${displayName} expects ${expectedArgsLength} argument`,
             `Function ${displayName} expects ${expectedArgsLength} arguments`,
             expectedArgsLength,
           ),
+          expression.node,
         );
       }
     }

frontend/src/metabase/query_builder/components/expressions/ExpressionEditorTextfield.jsx

@@ -253,10 +253,7 @@ export default class ExpressionEditorTextfield extends React.Component {
 
     this.clearSuggestions();
 
-    const { query, startRule } = this.props;
-    const { source } = this.state;
-
-    const errorMessage = diagnose(source, startRule, query);
+    const errorMessage = this.diagnoseExpression();
     this.setState({ errorMessage });
 
     // whenever our input blurs we push the updated expression to our parent if valid
@@ -315,6 +312,15 @@ export default class ExpressionEditorTextfield extends React.Component {
     return expression;
   }
 
+  diagnoseExpression() {
+    const { source } = this.state;
+    if (!source || source.length === 0) {
+      return { message: "Empty expression" };
+    }
+    const { query, startRule } = this.props;
+    return diagnose(source, startRule, query);
+  }
+
   commitExpression() {
     const { query, startRule } = this.props;
     const { source } = this.state;

frontend/test/metabase/lib/expressions/pratt/common.ts

+import {
+  lexify,
+  parse,
+  compile as newCompile,
+  Expr,
+} from "metabase/lib/expressions/pratt";
+import { resolve } from "metabase/lib/expressions/resolver";
+import { getMBQLName } from "metabase/lib/expressions/config";
+import {
+  parse as oldParser,
+  useShorthands,
+  adjustCase,
+  adjustOptions,
+} from "metabase/lib/expressions/recursive-parser";
+
+import { generateExpression } from "../generator";
+
+type Type = "expression" | "boolean";
+
+interface Opts {
+  throwOnError?: boolean;
+  resolverPass?: boolean;
+}
+
+export function compile(source: string, type: Type, opts: Opts = {}) {
+  const { throwOnError } = opts;
+  const passes = [adjustOptions, useShorthands, adjustCase];
+  return newCompile(
+    parse(lexify(source), {
+      throwOnError,
+    }).root,
+    {
+      passes: opts.resolverPass
+        ? [...passes, expr => resolve(expr, type, mockResolve as any)]
+        : passes,
+      getMBQLName,
+    },
+  );
+}
+
+export function mockResolve(kind: any, name: string): Expr {
+  return ["dimension", name];
+}
+
+export function oracle(source: string, type: Type) {
+  let mbql = null;
+  try {
+    mbql = oldParser(source);
+  } catch (e) {
+    let err = e as any;
+    if (err.length && err.length > 0) {
+      err = err[0];
+      if (typeof err.message === "string") {
+        err = err.message;
+      }
+    }
+    throw err;
+  }
+  return resolve(mbql, type, mockResolve as any);
+}
+
+export function compare(
+  source: string,
+  type: Type,
+  opts: Opts = {},
+): { oracle: any; compiled: any } {
+  const _oracle = oracle(source, type);
+  const compiled = compile(source, type, opts);
+  return { oracle: _oracle, compiled };
+}
+
+export function compareSeed(
+  seed: number,
+  type: Type,
+  opts: Opts = {},
+): { oracle: any; compiled: any } {
+  const { expression } = generateExpression(seed, type);
+  const _oracle = oracle(expression, type);
+  const compiled = compile(expression, type, opts);
+  return { oracle: _oracle, compiled };
+}

frontend/test/metabase/lib/expressions/pratt/compiler.unit.spec.ts

+import { compare, compile } from "./common";
+
+describe("metabase/lib/expressions/compiler", () => {
+  function expr(
+    source: string,
+    opts: { throwOnError?: boolean; resolverPass?: boolean } = {},
+  ) {
+    const { throwOnError = true, resolverPass } = opts;
+    return compile(source, "expression", { throwOnError, resolverPass });
+  }
+
+  describe("(for an expression)", () => {
+    it("should compile literals", () => {
+      expect(expr("42")).toEqual(42);
+      expect(expr("'Universe'")).toEqual("Universe");
+    });
+    /// TODO: Fix w/ some type info
+    it("should compile dimensions", () => {
+      expect(expr("[Price]")).toEqual(["dimension", "Price"]);
+      expect(expr("([X])")).toEqual(["dimension", "X"]);
+    });
+    it("should compile arithmetic operations", () => {
+      expect(expr("1+2")).toEqual(["+", 1, 2]);
+      expect(expr("3-4")).toEqual(["-", 3, 4]);
+      expect(expr("5*6")).toEqual(["*", 5, 6]);
+      expect(expr("7/8")).toEqual(["/", 7, 8]);
+      expect(expr("-(1+2)")).toEqual(["-", ["+", 1, 2]]);
+    });
+    it("should compile comparisons", () => {
+      expect(expr("1<2")).toEqual(["<", 1, 2]);
+      expect(expr("3>4")).toEqual([">", 3, 4]);
+      expect(expr("5<=6")).toEqual(["<=", 5, 6]);
+      expect(expr("7>=8")).toEqual([">=", 7, 8]);
+      expect(expr("9=9")).toEqual(["=", 9, 9]);
+      expect(expr("9!=0")).toEqual(["!=", 9, 0]);
+    });
+    it("should logical operators", () => {
+      expect(expr("7 or 8")).toEqual(["or", 7, 8]);
+      expect(expr("7 and 8")).toEqual(["and", 7, 8]);
+      expect(expr("7 and Size")).toEqual(["and", 7, ["dimension", "Size"]]);
+      expect(expr("NOT (7 and Size)")).toEqual([
+        "not",
+        ["and", 7, ["dimension", "Size"]],
+      ]);
+    });
+    it("should handle parenthesized expression", () => {
+      expect(expr("(42)")).toEqual(42);
+      expect(expr("-42")).toEqual(-42);
+      expect(expr("-(42)")).toEqual(["-", 42]);
+      expect(expr("((43))")).toEqual(43);
+      expect(expr("('Universe')")).toEqual("Universe");
+      expect(expr("(('Answer'))")).toEqual("Answer");
+      expect(expr("(1+2)")).toEqual(["+", 1, 2]);
+      expect(expr("(1+2)/3")).toEqual(["/", ["+", 1, 2], 3]);
+      expect(expr("4-(5*6)")).toEqual(["-", 4, ["*", 5, 6]]);
+      expect(expr("func_name(5*6, 4-3)")).toEqual([
+        "func_name",
+        ["*", 5, 6],
+        ["-", 4, 3],
+      ]);
+    });
+  });
+
+  describe("Should match the old compiler", () => {
+    it("Seed 59793: NOT NOT [p]<0", () => {
+      expect(expr("NOT NOT [p] < 0")).toEqual([
+        "not",
+        ["not", ["<", ["dimension", "p"], 0]],
+      ]);
+    });
+
+    it("Seed 59809: NOT ( ( [gG9_r]) )  >=( [__] )", () => {
+      expect(expr("NOT ( ( [gG9_r]) )  >=( [__] )")).toEqual([
+        "not",
+        [">=", ["dimension", "gG9_r"], ["dimension", "__"]],
+      ]);
+    });
+
+    it(`Seed 10099: CONtAinS ( [OF4wuV], SUbstriNG("_", -872096.613705, lENGtH("s Mfg7" ) ) )`, () => {
+      const {
+        oracle,
+        compiled,
+      } = compare(
+        `CONtAinS ( [OF4wuV], SUbstriNG("_", -872096.613705, lENGtH("s Mfg7" ) ) )`,
+        "boolean",
+        { throwOnError: true, resolverPass: false },
+      );
+      expect(compiled).toEqual(oracle);
+    });
+
+    it(`Seed 10092:  NOT ( NOT   (isNUll ([T0q → n_M_O])))`, () => {
+      const { oracle, compiled } = compare(
+        ` NOT ( NOT   (isNUll ([T0q → n_M_O])))`,
+        "expression",
+        {
+          throwOnError: true,
+          resolverPass: false,
+        },
+      );
+      expect(compiled).toEqual(oracle);
+    });
+
+    it(`Seed 10082: SUbstriNg( cOncat("BaK2    ", [__m_4], rTrim(coNcAt (replACE ([Av5Wtbz], regeXextRACt( [_1I → g], "H NVB84_"), rEGexextract ( [__8], " _ 2" )  ) , SUbStRiNG("qb0  ", (power( LeNgTh ( rtrim ( "YyCe_2" )) * 0e+77, 1 ) ), 1 + 0e-54 / 374719e-64) , cOncaT( "    F9 _O", "_a5_", " 5 _U_ ", " bE", rEPlACe (BXj3O, " ", [Z → X9]) ) )  )  ), (1), log ( 1E-26 )  ) `, () => {
+      const {
+        oracle,
+        compiled,
+      } = compare(
+        `SUbstriNg( cOncat("BaK2    ", [__m_4], rTrim(coNcAt (replACE ([Av5Wtbz], regeXextRACt( [_1I → g], "H NVB84_"), rEGexextract ( [__8], " _ 2" )  ) , SUbStRiNG("qb0  ", (power( LeNgTh ( rtrim ( "YyCe_2" )) * 0e+77, 1 ) ), 1 + 0e-54 / 374719e-64) , cOncaT( "    F9 _O", "_a5_", " 5 _U_ ", " bE", rEPlACe (BXj3O, " ", [Z → X9]) ) )  )  ), (1), log ( 1E-26 )  )`,
+        "expression",
+        { throwOnError: true, resolverPass: false },
+      );
+      expect(compiled).toEqual(oracle);
+    });
+
+    it(`Seed 57808:  (( Abs (  (exP( cEil(  - 1e+48) )) )  * -1e31 ) * ( poWeR( (( - 0e+67) *lengTh ( "8" )  ) ,  ( -1 ) *lengTh( "Q  P2c n" ) / powEr(1, N) ) )  )`, () => {
+      const {
+        oracle,
+        compiled,
+      } = compare(
+        `(( Abs (  (exP( cEil(  - 1e+48) )) )  * -1e31 ) * ( poWeR( (( - 0e+67) *lengTh ( "8" )  ) ,  ( -1 ) *lengTh( "Q  P2c n" ) / powEr(1, N) ) )  )`,
+        "expression",
+        { throwOnError: true, resolverPass: false },
+      );
+      expect(compiled).toEqual(oracle);
+    });
+
+    // Checks that `x - (y - z)` doesn't get merged into `x - y -z`
+    it(`seed 10144: eNDsWith( "NTP", replacE( [V2FFf → r_8ZFu], coalescE(repLACe([cf → l], sUbStriNg ( Replace( [A], "L ", "b"), coAlEsCE(953925E-38, 307355.510173e+32 ), pOwEr (1e+15, 0 )) , caSe ( IsEMpTy( [_ → _3H_6b]) , cOncat ("n_F e_B n" ) , isEmptY([E → _3R6p6_]), conCat(" D 2h", " 4 9u ", "A_9_M_9_", " q _")) ) , RegExeXtRact ([_PI9], "K43s 6") ) , sUBstriNg (CaSE (intervAl( [_OU9c], - 632269.595767E-79, CoalESce ("u__0_71", "c ")), CoalesCe( suBstriNG ( "XPHC0 li_", 500924.700063e-10, 341369) )), - 1E+47 - (424024.827478-1 ), - 1) ) )`, () => {
+      const {
+        oracle,
+        compiled,
+      } = compare(
+        `(( Abs (  (exP( cEil(  - 1e+48) )) )  * -1e31 ) * ( poWeR( (( - 0e+67) *lengTh ( "8" )  ) ,  ( -1 ) *lengTh( "Q  P2c n" ) / powEr(1, N) ) )  )`,
+        "expression",
+        { throwOnError: true, resolverPass: false },
+      );
+      expect(compiled).toEqual(oracle);
+    });
+
+    it(`Merging subtraction`, () => {
+      const { oracle, compiled } = compare(`1 - (0 - -10)`, "expression", {
+        throwOnError: true,
+      });
+      expect(compiled).toEqual(oracle);
+    });
+  });
+});

frontend/test/metabase/lib/expressions/pratt/parser.unit.spec.ts

+import { lexify, parse, Node } from "metabase/lib/expressions/pratt";
+
+describe("metabase/lib/expressions/parser", () => {
+  interface AST {
+    token: string;
+    children: AST[];
+    pos: number;
+  }
+  function cleanupAST(node: Node): AST {
+    return {
+      token: node.token?.text || "UNKNOWN",
+      children: node.children.map(cleanupAST),
+      pos: node.token?.pos || -1,
+    };
+  }
+
+  function parseExpression(source: string, throwOnError: boolean = true) {
+    return cleanupAST(
+      parse(lexify(source), {
+        throwOnError,
+      }).root,
+    );
+  }
+
+  const parseAggregation = parseExpression;
+  const parseFilter = parseExpression;
+
+  describe("Handles crazy expressions", () => {
+    it("Seed 66120", () => {
+      const expr = `(Z8YZP(_1(- CW(182751, (_d_YU_((h3M_))), 0e31, 0) > _d((0e+96) >= [UVz], NOT ltKvh([__7g], (\"85\")) / - (D___(h627I1 (50106.e80, Qh8_U(([B] = \"d9 \"), 586107.E1 OR [__42 → G_TH]), [Cc_]) + 22882e+22, \"ws\" * 642705.632614e+47, - _2Y_nZi + - __k1_q_(((519470.E+61)), [D]) >= (0E20), [__1sCC3 → mz8])), (M(Q4__L(), (NOT NOT NOT NOT - \"g\" + NOT 0e+35))) <= 798469.324524E54)), h((- R8__K), T6([f3w67], NOT [D8d2933])), ftI7YA(0E+10 <= [F1Y_ → tuj] * 907239. = (NOT O1sW7pq((631184.576387e+12 > _H___3B(- [v_h_4])) != - z) <= ([_EvzL_8 → Ar]) <= [e1_Jy57]) AND _u2_(252719, \" 9\"), __(hB(NOT \"7j 1_V_\", 660445E+96, (x0E9ox(k__p(_9h5_() AND e(\"70_f\", __2_S_Q()) + [__W6U → L7UnD6J] = (NOT NOT N(h_9(\"4 g \", 575068.), NOT \"487\", [S → yWQ_], (\"  \" + \" k_ dd2d \"))), (TMIVF(\"c\", - [_gx3sJ → g___5j], (559929.E66))), - [jUc_y → o7], p84__t8((((Mg8_()))), - [ZF59_w8 → U], - [T_ → gw_2k3] OR (848268.635035E+38 * (20814.e-38)), P_7_0 <= y09(C3_(399920), Lw_O(- 252450.180012e-81 = 290101.992467, 71543.409973E65, A((_()), H(858127., 1, \"_W D\", \" __\"), - 253307.244574e21, [IZ_Q_D])), _q(\"_ _\"), rRa2) > Q(_1Ux0, NOT K__j89, - (V_7IS), \"_n58\"))))) - 719454), sq_R(), 346932., NOT B___())), - 787799.)) < \"712\"`;
+      expect(() => parseExpression(expr)).not.toThrow();
+    });
+
+    it("Seed 635496", () => {
+      const expr = `K__(m_AA(yyg6_t((NOT [XH8]), [W → PU]) > [k8xNd3 → md_zfIs] + a9_H9MP, a___tf(p_G(- C_Ug((_5iP), (vh_) > _zaS_, 578833.e+0 = ([_9S_64] <= C_(t53I, Q_6(NOT MO_B(0, NOT K61p_f)))))), (866983.)), __E(_W7a_, oXm7A(_01E_(627139.300685 > NOT u_1_d6(p4564j1(gZfe_9((- 727986.), p7hU()) * - - U2w_k9_(0, 1, \"69 a\", \" \"), _129)) <= NOT \"k_K\", - (_chTf5V() AND E3_(u_X2_, (NOT qImu5()), 955004) = ZQcm <= 96160.580572), 904595.), 46320.407574E+99) > [hXw → H_KD], NOT 126223.561027, (C_I92O) OR (NOT Je5_C([UvO_b1 → _i_S], 0, 647472E+81 != - __(), 920107.)) != (NOT f(68493., 335976.415413 > (1))) != _(_y(\"___\", [___u_L], [_w], [ODVlD → nM])) > \"Qq \"), ((\"7\") * 754944.E85)) >= (- XzB__(t1782, (\"r_\") <= \"__wa\") AND - 107898.37690E+16)) = (NOT [J_ → _])`;
+      expect(() => parseExpression(expr)).not.toThrow();
+    });
+
+    it("Seed 59999", () => {
+      const expr = `_z * NOT 0 = TS(__O_v2_(_t(- (- Y_), (NOT _27a != NOT T80Bm67((20804.), 664158.926033 AND [B82PH_], - NOT (891819 < \"a_wf_DI\") - 731090. >= 1 <= ((_g0AC(867927E+79)) <= \"_P \" = 990643.), ([yL4P_0 → _])) >= gn7_19(0 AND um6(((((202340.)))), NOT \"P\", (0 + - \"_l4k3\" = (_HG7_B(\"8t5_s _O2\", 1, \"Y_d_\", 0)))), [L2_], NOT 648865.E73, - - (355186e+44)))), [_G], [__Z → G_U9], NOT (NOT 756239.969634E-67 = ((_baX_r())) + ((NOT - [Q4 → pB]))) AND f__ErS(766747, [hc0y → sjA_h], [_ → v7__kJ1] * (D1_n18(_2B(1))) AND NOT u7(vj__) OR 847581, NOT (1e+5))), \"WZztF\")`;
+      expect(() => parseExpression(expr)).not.toThrow();
+    });
+  });
+
+  describe("handles fields", () => {
+    // ---- Negation - number ----
+    it("should accept a field", () => {
+      expect(() => parseExpression("[A]")).not.toThrow();
+    });
+    it("should accept a negative field", () => {
+      expect(() => parseExpression("-[A]")).not.toThrow();
+    });
+
+    it("should accept a field with escaped parentheses", () => {
+      expect(() => parseExpression("[A \\(B\\)]")).not.toThrow();
+    });
+
+    it("should accept an escaped field", () => {
+      expect(() => parseExpression("[A \\[T\\]]")).not.toThrow();
+    });
+  });
+
+  describe("handles negation", () => {
+    // ---- Negation - number ----
+    it("should accept a negative number", () => {
+      expect(() => parseExpression("-42")).not.toThrow();
+    });
+    it("should accept a negative number (outside parentheses)", () => {
+      expect(() => parseExpression("-(42)")).not.toThrow();
+    });
+    it("should accept a negative number (inside parentheses)", () => {
+      expect(() => parseExpression("(-42)")).not.toThrow();
+    });
+
+    // ---- Negation - identifier ----
+    it("should accept a negative identifier", () => {
+      expect(() => parseExpression("-[X]")).not.toThrow();
+    });
+    it("should accept a negative identifier (outside parentheses)", () => {
+      expect(() => parseExpression("-([X])")).not.toThrow();
+    });
+    it("should accept a negative identifier (inside parentheses)", () => {
+      expect(() => parseExpression("(-[X])")).not.toThrow();
+    });
+
+    // ---- Negation - identifier ----
+    it("should accept a negative function call", () => {
+      expect(() => parseExpression("-abs([X])")).not.toThrow();
+    });
+    it("should accept a negative function call (outside parentheses)", () => {
+      expect(() => parseExpression("-(abs([X]))")).not.toThrow();
+    });
+    it("should accept a negative function call (inside parentheses)", () => {
+      expect(() => parseExpression("(-abs([X]))")).not.toThrow();
+    });
+
+    /// ---- Other weird negations ----
+    it("should accept a double negation with syntax error", () => {
+      expect(() => parseExpression("NOT NOT Or", false)).not.toThrow();
+    });
+  });
+
+  describe("(in expression mode)", () => {
+    it("should accept a number", () => {
+      expect(() => parseExpression("42")).not.toThrow();
+    });
+    it("should accept a single-quoted string", () => {
+      expect(() => parseExpression("'Answer'")).not.toThrow();
+    });
+    it("should accept an escaped single-quoted string", () => {
+      expect(() => parseExpression("'An\\'swer'")).not.toThrow();
+    });
+    it("should accept a double-quoted string", () => {
+      expect(() => parseExpression('"Answer"')).not.toThrow();
+    });
+    it("should accept an escaped double-quoted string", () => {
+      expect(() => parseExpression('"An\\"swer"')).not.toThrow();
+    });
+    it("should accept a group expression (in parentheses)", () => {
+      expect(() => parseExpression("(42)")).not.toThrow();
+    });
+
+    it("should accept the function lower", () => {
+      expect(() => parseExpression("Lower([Title])")).not.toThrow();
+    });
+    it("should accept the function upper", () => {
+      expect(() => parseExpression("Upper([Title])")).not.toThrow();
+    });
+
+    it("should accept the function CASE", () => {
+      expect(() => parseExpression("Case([Z]>7, 'X', 'Y')")).not.toThrow();
+    });
+    it("should accept the function CASE with multiple cases", () => {
+      expect(() => parseExpression("Case([X]>5,5,[X]>3,3,0)")).not.toThrow();
+    });
+
+    it("should reject an unclosed single-quoted string", () => {
+      expect(() => parseExpression('"Answer')).toThrow();
+    });
+    it("should reject an unclosed double-quoted string", () => {
+      expect(() => parseExpression('"Answer')).toThrow();
+    });
+    it("should reject a mismatched quoted string", () => {
+      expect(() => parseExpression("\"Answer'")).toThrow();
+    });
+    it("should handle a conditional with ISEMPTY", () => {
+      expect(() =>
+        parseExpression("case(isempty([Discount]),[P])"),
+      ).not.toThrow();
+    });
+    // TODO: This should be handled by a separate pass
+    xit("should reject CASE with only one argument", () => {
+      expect(() => parseExpression("case([Deal])")).toThrow();
+    });
+    it("should accept CASE with two arguments", () => {
+      expect(() => parseExpression("case([Deal],x)")).not.toThrow();
+    });
+    it("should reject CASE missing a closing paren", () => {
+      expect(() => parseExpression("case([Deal],x")).toThrow();
+    });
+  });
+
+  describe("(in aggregation mode)", () => {
+    it("should accept an aggregration with COUNT", () => {
+      expect(() => parseAggregation("Count()")).not.toThrow();
+    });
+    it("should accept an aggregration with SUM", () => {
+      expect(() => parseAggregation("Sum([Price])")).not.toThrow();
+    });
+    it("should accept an aggregration with DISTINCT", () => {
+      expect(() => parseAggregation("Distinct([Supplier])")).not.toThrow();
+    });
+    it("should accept an aggregration with STANDARDDEVIATION", () => {
+      expect(() => parseAggregation("StandardDeviation([Debt])")).not.toThrow();
+    });
+    it("should accept an aggregration with AVERAGE", () => {
+      expect(() => parseAggregation("Average([Height])")).not.toThrow();
+    });
+    it("should accept an aggregration with MAX", () => {
+      expect(() => parseAggregation("Max([Discount])")).not.toThrow();
+    });
+    it("should accept an aggregration with MIN", () => {
+      expect(() => parseAggregation("Min([Rating])")).not.toThrow();
+    });
+    it("should accept an aggregration with MEDIAN", () => {
+      expect(() => parseAggregation("Median([Total])")).not.toThrow();
+    });
+    it("should accept an aggregration with VAR", () => {
+      expect(() => parseAggregation("Variance([Tax])")).not.toThrow();
+    });
+
+    it("should accept a conditional aggregration with COUNTIF", () => {
+      expect(() => parseAggregation("CountIf([Discount] > 0)")).not.toThrow();
+    });
+
+    it("should accept a conditional aggregration with COUNTIF containing an expression", () => {
+      expect(() => parseAggregation("CountIf(([A]+[B]) > 1)")).not.toThrow();
+      expect(() =>
+        parseAggregation("CountIf( 1.2 * [Price] > 37)"),
+      ).not.toThrow();
+    });
+  });
+
+  describe("(in filter mode)", () => {
+    it("should accept a simple comparison", () => {
+      expect(() => parseFilter("[Total] > 12")).not.toThrow();
+    });
+    it("should accept another simple comparison", () => {
+      expect(() => parseFilter("10 < [DiscountPercent]")).not.toThrow();
+    });
+    it("should accept a logical NOT", () => {
+      expect(() => parseFilter("NOT [Debt] > 5")).not.toThrow();
+    });
+    it("should accept a segment", () => {
+      expect(() => parseFilter("[SpecialDeal]")).not.toThrow();
+    });
+    it("should accept a logical NOT on segment", () => {
+      expect(() => parseFilter("NOT [Clearance]")).not.toThrow();
+    });
+    it("should accept multiple logical NOTs on segment", () => {
+      expect(() => parseFilter("NOT NOT [Clearance]")).not.toThrow();
+    });
+    it("should accept a relational between a segment and a dimension", () => {
+      expect(() => parseFilter("([Shipping] < 2) AND [Sale]")).not.toThrow();
+    });
+    it("should accept parenthesized logical operations", () => {
+      expect(() => parseFilter("([Deal] AND [HighRating])")).not.toThrow();
+      expect(() => parseFilter("([Price] < 100 OR [Refurb])")).not.toThrow();
+    });
+    it("should accept a function", () => {
+      expect(() => parseFilter("between([Subtotal], 1, 2)")).not.toThrow();
+    });
+
+    // TODO: This should be handled by a separate pass
+    xit("should reject CASE with only one argument", () => {
+      expect(() => parseFilter("case([Deal])")).toThrow();
+    });
+  });
+});

frontend/test/metabase/scenarios/custom-column/cc-expression-editor.cy.spec.js

@@ -49,7 +49,7 @@ describe("scenarios > question > custom column > expression editor", () => {
       .type("{movetoend}{backspace}", { force: true })
       .blur();
 
-    cy.findByText("Unexpected end of input");
+    cy.findByText("Expected expression");
     cy.button("Done").should("be.disabled");
   });