From 449d5d3bcb9127076348d299cd6e46b1a4c4817c Mon Sep 17 00:00:00 2001
From: Ariya Hidayat <ariya@metabase.com>
Date: Mon, 15 Nov 2021 07:41:17 -0800
Subject: [PATCH] Custom expression tokenizer: rewind on an unterminated string
literal (#18969)
---
.../src/metabase/lib/expressions/tokenizer.js | 15 +++++++++++----
.../lib/expressions/completer.unit.spec.js | 2 --
.../lib/expressions/tokenizer.unit.spec.js | 12 ++++++++++++
3 files changed, 23 insertions(+), 6 deletions(-)
diff --git a/frontend/src/metabase/lib/expressions/tokenizer.js b/frontend/src/metabase/lib/expressions/tokenizer.js
index c627187980e..c5a6449f18b 100644
--- a/frontend/src/metabase/lib/expressions/tokenizer.js
+++ b/frontend/src/metabase/lib/expressions/tokenizer.js
@@ -215,10 +215,17 @@ export function tokenize(expression) {
}
}
const type = TOKEN.String;
- const end = index;
- const terminated = quote === source[end - 1];
- const error = terminated ? null : t`Missing closing quotes`;
- return { type, value, start, end, error };
+ let error = null;
+
+ const terminated = quote === source[index - 1];
+ if (!terminated) {
+ // unterminated string, rewind after the opening quote
+ index = start + 1;
+ value = quote;
+ error = t`Missing closing quotes`;
+ }
+
+ return { type, value, start, end: index, error };
};
const scanBracketIdentifier = () => {
diff --git a/frontend/test/metabase/lib/expressions/completer.unit.spec.js b/frontend/test/metabase/lib/expressions/completer.unit.spec.js
index 59e0e23c23c..1c876fe3418 100644
--- a/frontend/test/metabase/lib/expressions/completer.unit.spec.js
+++ b/frontend/test/metabase/lib/expressions/completer.unit.spec.js
@@ -22,8 +22,6 @@ describe("metabase/lib/expressions/completer", () => {
expect(partialMatch("X OR")).toEqual(null);
expect(partialMatch("42 +")).toEqual(null);
expect(partialMatch("3.14")).toEqual(null);
- expect(partialMatch('"Hello')).toEqual(null);
- expect(partialMatch("'world")).toEqual(null);
});
it("should handle empty input", () => {
diff --git a/frontend/test/metabase/lib/expressions/tokenizer.unit.spec.js b/frontend/test/metabase/lib/expressions/tokenizer.unit.spec.js
index 91a2163cbe8..4bf1d977fed 100644
--- a/frontend/test/metabase/lib/expressions/tokenizer.unit.spec.js
+++ b/frontend/test/metabase/lib/expressions/tokenizer.unit.spec.js
@@ -66,6 +66,18 @@ describe("metabase/lib/expressions/tokenizer", () => {
expect(errors('"double')[0].message).toEqual("Missing closing quotes");
});
+ it("should continue to tokenize when encountering an unterminated string literal", () => {
+ expect(types("CONCAT(universe') = [answer]")).toEqual([
+ T.Identifier,
+ T.Operator,
+ T.Identifier,
+ T.String,
+ T.Operator,
+ T.Operator,
+ T.Identifier,
+ ]);
+ });
+
it("should tokenize identifiers", () => {
expect(types("Price")).toEqual([T.Identifier]);
expect(types("Special_Deal")).toEqual([T.Identifier]);
--
GitLab