From 14d2328c88317b0ff92b6468f6e3e3e2d5553ee7 Mon Sep 17 00:00:00 2001
From: Samuel Degueldre <sad@odoo.com>
Date: Tue, 8 Mar 2022 09:37:09 +0100
Subject: [PATCH] [IMP] compiler: improve error message for tokenization errors

---
 src/compiler/inline_expressions.ts        | 32 ++++++++++++++---------
 tests/compiler/inline_expressions.test.ts | 10 +++----
 2 files changed, 24 insertions(+), 18 deletions(-)

diff --git a/src/compiler/inline_expressions.ts b/src/compiler/inline_expressions.ts
index 5840d980..5cde7949 100644
--- a/src/compiler/inline_expressions.ts
+++ b/src/compiler/inline_expressions.ts
@@ -199,24 +199,30 @@ const TOKENIZERS = [
 export function tokenize(expr: string): Token[] {
   const result: Token[] = [];
   let token: boolean | Token = true;
+  let error: any;
+  let current = expr;
 
-  while (token) {
-    expr = expr.trim();
-    if (expr) {
-      for (let tokenizer of TOKENIZERS) {
-        token = tokenizer(expr);
-        if (token) {
-          result.push(token);
-          expr = expr.slice(token.size || token.value.length);
-          break;
+  try {
+    while (token) {
+      current = current.trim();
+      if (current) {
+        for (let tokenizer of TOKENIZERS) {
+          token = tokenizer(current);
+          if (token) {
+            result.push(token);
+            current = current.slice(token.size || token.value.length);
+            break;
+          }
         }
+      } else {
+        token = false;
       }
-    } else {
-      token = false;
     }
+  } catch (e) {
+    error = e; // Silence all errors and throw a generic error below
   }
-  if (expr.length) {
-    throw new Error(`Tokenizer error: could not tokenize "${expr}"`);
+  if (current.length || error) {
+    throw new Error(`Tokenizer error: could not tokenize \`${expr}\``);
   }
   return result;
 }
diff --git a/tests/compiler/inline_expressions.test.ts b/tests/compiler/inline_expressions.test.ts
index 4743854a..ae6fe234 100644
--- a/tests/compiler/inline_expressions.test.ts
+++ b/tests/compiler/inline_expressions.test.ts
@@ -65,14 +65,14 @@ describe("tokenizer", () => {
   });
 
   test("strings", () => {
-    expect(() => tokenize("'")).toThrow("Invalid expression");
-    expect(() => tokenize("'\\")).toThrow("Invalid expression");
-    expect(() => tokenize("'\\'")).toThrow("Invalid expression");
+    expect(() => tokenize("'")).toThrow("Tokenizer error: could not tokenize `'`");
+    expect(() => tokenize("'\\")).toThrow("Tokenizer error: could not tokenize `'\\`");
+    expect(() => tokenize("'\\'")).toThrow("Tokenizer error: could not tokenize `'\\'`");
     expect(tokenize("'hello ged'")).toEqual([{ type: "VALUE", value: "'hello ged'" }]);
     expect(tokenize("'hello \\'ged\\''")).toEqual([{ type: "VALUE", value: "'hello \\'ged\\''" }]);
 
-    expect(() => tokenize('"')).toThrow("Invalid expression");
-    expect(() => tokenize('"\\"')).toThrow("Invalid expression");
+    expect(() => tokenize('"')).toThrow('Tokenizer error: could not tokenize `"`');
+    expect(() => tokenize('"\\"')).toThrow('Tokenizer error: could not tokenize `"\\"`');
     expect(tokenize('"hello ged"')).toEqual([{ type: "VALUE", value: '"hello ged"' }]);
     expect(tokenize('"hello ged"}')).toEqual([
       { type: "VALUE", value: '"hello ged"' },