Custom expression suggestion: refactor the logic for partial matching (#15889)

Add some more extensive unit tests.

* Test for operator, not whitespace

frontend/src/metabase/lib/expressions/completer.js

+import { tokenize, TOKEN } from "./tokenizer";
+import _ from "underscore";
+
+// Given an expression, get the last identifier as the prefix match.
+// Examples:
+//  "Lower" returns "Lower"
+//  "3 > [Rat" returns "[Rat"
+//  "[Expensive] " returns null (because of the whitespace)
+//  "IsNull([Tax])" returns null (last token is an operator)
+
+export function partialMatch(expression) {
+  const { tokens } = tokenize(expression);
+  const lastToken = _.last(tokens);
+  if (lastToken && lastToken.type === TOKEN.Identifier) {
+    if (lastToken.end === expression.length) {
+      return expression.slice(lastToken.start, lastToken.end);
+    }
+  }
+
+  return null;
+}

frontend/src/metabase/lib/expressions/suggest.js

@@ -28,11 +28,12 @@ import {
   Minus,
   NumberLiteral,
   StringLiteral,
-  UnclosedQuotedString,
   isTokenType,
   lexerWithRecovery,
 } from "./lexer";
 
+import { partialMatch } from "./completer";
+
 import getHelpText from "./helper_text_strings";
 
 import { ExpressionDimension } from "metabase-lib/lib/Dimension";
@@ -70,24 +71,12 @@ export function suggest({
   if (lexResult.errors.length > 0) {
     throw lexResult.errors;
   }
-
-  let partialSuggestionMode = false;
   let tokenVector = lexResult.tokens;
 
-  const lastInputToken = _.last(lexResult.tokens);
-  const lastInputTokenIsUnclosedIdentifierString =
-    lastInputToken &&
-    isTokenType(lastInputToken.tokenType, UnclosedQuotedString) &&
-    isTokenType(lastInputToken.tokenType, IdentifierString);
-  // we have requested assistance while inside an Identifier or Unclosed IdentifierString
-  if (
-    lastInputToken &&
-    ((isTokenType(lastInputToken.tokenType, Identifier) &&
-      Identifier.PATTERN.test(partialSource[partialSource.length - 1])) ||
-      lastInputTokenIsUnclosedIdentifierString)
-  ) {
+  const matchPrefix = partialMatch(partialSource);
+  const partialSuggestionMode = matchPrefix && matchPrefix.length > 0;
+  if (partialSuggestionMode) {
     tokenVector = tokenVector.slice(0, -1);
-    partialSuggestionMode = true;
   }
 
   const context = getContext({
@@ -185,8 +174,6 @@ export function suggest({
           })),
         );
       }
-    } else if (lastInputTokenIsUnclosedIdentifierString) {
-      // skip the rest
     } else if (
       isTokenType(nextTokenType, FunctionName) ||
       nextTokenType === Case
@@ -251,10 +238,7 @@ export function suggest({
 
   // throw away any suggestion that is not a suffix of the last partialToken.
   if (partialSuggestionMode) {
-    const input = lastInputToken.image;
-    const partial = lastInputTokenIsUnclosedIdentifierString
-      ? input.slice(1).toLowerCase()
-      : input.toLowerCase();
+    const partial = matchPrefix.toLowerCase();
     for (const suggestion of finalSuggestions) {
       suggestion: for (const text of [
         suggestion.name,

frontend/test/metabase/lib/expressions/completer.unit.spec.js

+import { partialMatch } from "metabase/lib/expressions/completer";
+
+describe("metabase/lib/expressions/completer", () => {
+  describe("partialMatch", () => {
+    it("should get the function name", () => {
+      expect(partialMatch("Lowe")).toEqual("Lowe");
+      expect(partialMatch("NOT (ISNULL")).toEqual("ISNULL");
+    });
+
+    it("should get the field name", () => {
+      expect(partialMatch("[Deal]")).toEqual("[Deal]");
+      expect(partialMatch("A")).toEqual("A");
+      expect(partialMatch("B AND [Ca")).toEqual("[Ca");
+      expect(partialMatch("[Sale] or [Good]")).toEqual("[Good]");
+      expect(partialMatch("[")).toEqual("[");
+    });
+
+    it("should ignore operators and literals", () => {
+      expect(partialMatch("X OR")).toEqual(null);
+      expect(partialMatch("42 +")).toEqual(null);
+      expect(partialMatch("3.14")).toEqual(null);
+      expect(partialMatch('"Hello')).toEqual(null);
+      expect(partialMatch("'world")).toEqual(null);
+    });
+
+    it("should handle empty input", () => {
+      expect(partialMatch("")).toEqual(null);
+      expect(partialMatch(" ")).toEqual(null);
+    });
+  });
+});