Custom expression tokenizer: rewind on an unterminated string literal (#18969)

frontend/src/metabase/lib/expressions/tokenizer.js

@@ -215,10 +215,17 @@ export function tokenize(expression) {
       }
     }
     const type = TOKEN.String;
-    const end = index;
-    const terminated = quote === source[end - 1];
-    const error = terminated ? null : t`Missing closing quotes`;
-    return { type, value, start, end, error };
+    let error = null;
+
+    const terminated = quote === source[index - 1];
+    if (!terminated) {
+      // unterminated string, rewind after the opening quote
+      index = start + 1;
+      value = quote;
+      error = t`Missing closing quotes`;
+    }
+
+    return { type, value, start, end: index, error };
   };
 
   const scanBracketIdentifier = () => {

frontend/test/metabase/lib/expressions/completer.unit.spec.js

@@ -22,8 +22,6 @@ describe("metabase/lib/expressions/completer", () => {
       expect(partialMatch("X OR")).toEqual(null);
       expect(partialMatch("42 +")).toEqual(null);
       expect(partialMatch("3.14")).toEqual(null);
-      expect(partialMatch('"Hello')).toEqual(null);
-      expect(partialMatch("'world")).toEqual(null);
     });
 
     it("should handle empty input", () => {

frontend/test/metabase/lib/expressions/tokenizer.unit.spec.js

@@ -66,6 +66,18 @@ describe("metabase/lib/expressions/tokenizer", () => {
     expect(errors('"double')[0].message).toEqual("Missing closing quotes");
   });
 
+  it("should continue to tokenize when encountering an unterminated string literal", () => {
+    expect(types("CONCAT(universe') = [answer]")).toEqual([
+      T.Identifier,
+      T.Operator,
+      T.Identifier,
+      T.String,
+      T.Operator,
+      T.Operator,
+      T.Identifier,
+    ]);
+  });
+
   it("should tokenize identifiers", () => {
     expect(types("Price")).toEqual([T.Identifier]);
     expect(types("Special_Deal")).toEqual([T.Identifier]);