[IMP] qweb: implements an expression evaluator

closes #5

doc/qweb.md

@@ -175,9 +175,15 @@ root nodes.
 
 ### Expression evaluation
 
-It is useful to explain the various rules that applies on QWeb expressions. These
+QWeb expressions are strings that will be processed at compile time. Each variable in
-expressions are strings that will be converted to a javascript expression at
+the javascript expression will be replaced by a lookup in the context (so, the
-compile time.
+widget).  For example, `a + b.c(d)` will be converted into:
+
+```js
+context['a'] + context['b'].c(context['d'])
+```
+
+It is useful to explain the various rules that applies on these expressions:
 
 1. it should be a simple expression which returns a value. It cannot be a statement.
 
@@ -191,10 +197,10 @@ compile time.
    <div><p t-if="console.log(1)">NOT valid</p></div>
    ```
 
-2. it can use anything in the rendering context:
+2. it can use anything in the rendering context (typically, the component):
 
    ```xml
-   <p t-if="user.birthday == today()">Happy bithday!</p>
+   <p t-if="user.birthday === today()">Happy bithday!</p>
    ```
 
    is valid, and will read the `user` object from the context, and call the

src/qweb_core.ts

@@ -1,4 +1,5 @@
 import { VNode, h } from "./vdom";
+import { QWebVar, compileExpr } from "./qweb_expressions";
 
 /**
  * Owl QWeb Engine
@@ -56,18 +57,6 @@ export interface Directive {
 //------------------------------------------------------------------------------
 // Const/global stuff/helpers
 //------------------------------------------------------------------------------
-const RESERVED_WORDS = "true,false,NaN,null,undefined,debugger,console,window,in,instanceof,new,function,return,this,typeof,eval,void,Math,RegExp,Array,Object,Date".split(
-  ","
-);
-
-const WORD_REPLACEMENT = {
-  and: "&&",
-  or: "||",
-  gt: ">",
-  gte: ">=",
-  lt: "<",
-  lte: "<="
-};
 
 const DISABLED_TAGS = [
   "input",
@@ -624,16 +613,6 @@ export class QWeb {
 //------------------------------------------------------------------------------
 // Compilation Context
 //------------------------------------------------------------------------------
-export interface QWebExprVar {
-  id: string;
-  expr: string;
-}
-
-export interface QWebXMLVar {
-  xml: NodeList;
-}
-
-export type QWebVar = QWebExprVar | QWebXMLVar;
 
 export class Context {
   nextID: number = 1;
@@ -713,63 +692,14 @@ export class Context {
     return val in this.variables ? this.getValue(this.variables[val]) : val;
   }
 
-  formatExpression(e: string): string {
+  /**
-    e = e.trim();
+   * Prepare an expression for being consumed at render time.  Its main job
-    if (e[0] === "{" && e[e.length - 1] === "}") {
+   * is to
-      const innerExpr = e
+   * - replace unknown variables by a lookup in the context
-        .slice(1, -1)
+   * - replace already defined variables by their internal name
-        .split(",")
+   */
-        .map(p => {
+  formatExpression(expr: string): string {
-          let [key, val] = p.trim().split(":");
+    return compileExpr(expr, this.variables);
-          if (key === "") {
-            return "";
-          }
-          if (!val) {
-            val = key;
-          }
-          return `${key}: ${this.formatExpression(val)}`;
-        })
-        .join(",");
-      return "{" + innerExpr + "}";
-    }
-
-    // Thanks CHM for this code...
-    const chars = e.split("");
-    let instring = "";
-    let invar = "";
-    let invarPos = 0;
-    let r = "";
-    chars.push(" ");
-    for (let i = 0, ilen = chars.length; i < ilen; i++) {
-      let c = chars[i];
-      if (instring.length) {
-        if (c === instring && chars[i - 1] !== "\\") {
-          instring = "";
-        }
-      } else if (c === '"' || c === "'") {
-        instring = c;
-      } else if (c.match(/[a-zA-Z_\$]/) && !invar.length) {
-        invar = c;
-        invarPos = i;
-        continue;
-      } else if (c.match(/\W/) && invar.length) {
-        // TODO: Should check for possible spaces before dot
-        if (chars[invarPos - 1] !== "." && RESERVED_WORDS.indexOf(invar) < 0) {
-          invar =
-            WORD_REPLACEMENT[invar] ||
-            (this.variables[invar] && (<any>this.variables[invar]).id) ||
-            "context['" + invar + "']";
-        }
-        r += invar;
-        invar = "";
-      } else if (invar.length) {
-        invar += c;
-        continue;
-      }
-      r += c;
-    }
-    const result = r.slice(0, -1);
-    return result;
   }
 
   /**

src/qweb_directives.ts

@@ -1,4 +1,5 @@
-import { Context, QWeb, UTILS, QWebExprVar } from "./qweb_core";
+import { Context, QWeb, UTILS } from "./qweb_core";
+import { QWebExprVar } from "./qweb_expressions";
 
 /**
  * Owl QWeb Directives

src/qweb_expressions.ts

@@ -0,0 +1,270 @@
+/**
+ * Owl QWeb Expression Parser
+ *
+ * Owl needs in various contexts to be able to understand the structure of a
+ * string representing a javascript expression.  The usual goal is to be able
+ * to rewrite some variables.  For example, if a template has
+ *
+ *  ```xml
+ *  <t t-if="computeSomething({val: state.val})">...</t>
+ * ```
+ *
+ * this needs to be translated in something like this:
+ *
+ * ```js
+ *   if (context["computeSomething"]({val: context["state"].val})) { ... }
+ * ```
+ *
+ * This file contains the implementation of an extremely naive tokenizer/parser
+ * and evaluator for javascript expressions.  The supported grammar is basically
+ * only expressive enough to understand the shape of objects, of arrays, and
+ * various operators.
+ */
+
+//------------------------------------------------------------------------------
+// Misc types, constants and helpers
+//------------------------------------------------------------------------------
+
+const RESERVED_WORDS = "true,false,NaN,null,undefined,debugger,console,window,in,instanceof,new,function,return,this,typeof,eval,void,Math,RegExp,Array,Object,Date".split(
+  ","
+);
+
+const WORD_REPLACEMENT = {
+  and: "&&",
+  or: "||",
+  gt: ">",
+  gte: ">=",
+  lt: "<",
+  lte: "<="
+};
+
+export interface QWebExprVar {
+  id: string;
+  expr: string;
+}
+
+export interface QWebXMLVar {
+  xml: NodeList;
+}
+
+export type QWebVar = QWebExprVar | QWebXMLVar;
+
+//------------------------------------------------------------------------------
+// Tokenizer
+//------------------------------------------------------------------------------
+type TKind =
+  | "LEFT_BRACE"
+  | "RIGHT_BRACE"
+  | "LEFT_BRACKET"
+  | "RIGHT_BRACKET"
+  | "LEFT_PAREN"
+  | "RIGHT_PAREN"
+  | "COMMA"
+  | "VALUE"
+  | "SYMBOL"
+  | "OPERATOR"
+  | "COLON";
+
+interface Token {
+  type: TKind;
+  value: string;
+  size?: number;
+}
+
+const STATIC_TOKEN_MAP: { [key: string]: TKind } = {
+  "{": "LEFT_BRACE",
+  "}": "RIGHT_BRACE",
+  "[": "LEFT_BRACKET",
+  "]": "RIGHT_BRACKET",
+  ":": "COLON",
+  ",": "COMMA",
+  "(": "LEFT_PAREN",
+  ")": "RIGHT_PAREN"
+};
+
+const OPERATORS = [".", "===", "==", "+", "!", "||", "&&", ">", "?", "-", "*"];
+
+type Tokenizer = (expr: string) => Token | false;
+
+let tokenizeString: Tokenizer = function(expr) {
+  let s = expr[0];
+  let start = s;
+  if (s !== "'" && s !== '"') {
+    return false;
+  }
+  let i = 1;
+  let cur;
+  while (expr[i] && expr[i] !== start) {
+    cur = expr[i];
+    s += cur;
+    if (cur === "\\") {
+      i++;
+      cur = expr[i];
+      if (!cur) {
+        throw new Error("Invalid expression");
+      }
+      s += cur;
+    }
+    i++;
+  }
+  if (expr[i] !== start) {
+    throw new Error("Invalid expression");
+  }
+  s += start;
+  return { type: "VALUE", value: s };
+};
+
+let tokenizeNumber: Tokenizer = function(expr) {
+  let s = expr[0];
+  if (s && s.match(/[0-9]/)) {
+    let i = 1;
+    while (expr[i] && expr[i].match(/[0-9]|\./)) {
+      s += expr[i];
+      i++;
+    }
+    return { type: "VALUE", value: s };
+  } else {
+    return false;
+  }
+};
+
+let tokenizeSymbol: Tokenizer = function(expr) {
+  let s = expr[0];
+  if (s && s.match(/[a-zA-Z_\$]/)) {
+    let i = 1;
+    while (expr[i] && expr[i].match(/\w/)) {
+      s += expr[i];
+      i++;
+    }
+    if (s in WORD_REPLACEMENT) {
+      return { type: "OPERATOR", value: WORD_REPLACEMENT[s], size: s.length };
+    }
+    return { type: "SYMBOL", value: s };
+  } else {
+    return false;
+  }
+};
+
+const tokenizeStatic: Tokenizer = function(expr) {
+  const char = expr[0];
+  if (char && char in STATIC_TOKEN_MAP) {
+    return { type: STATIC_TOKEN_MAP[char], value: char };
+  }
+  return false;
+};
+
+const tokenizeOperator: Tokenizer = function(expr) {
+  for (let op of OPERATORS) {
+    if (expr.startsWith(op)) {
+      return { type: "OPERATOR", value: op };
+    }
+  }
+  return false;
+};
+
+const TOKENIZERS = [
+  tokenizeString,
+  tokenizeNumber,
+  tokenizeSymbol,
+  tokenizeStatic,
+  tokenizeOperator
+];
+
+/**
+ * Convert a javascript expression (as a string) into a list of tokens. For
+ * example: `tokenize("1 + b")` will return:
+ * ```js
+ *  [
+ *   {type: "VALUE", value: "1"},
+ *   {type: "OPERATOR", value: "+"},
+ *   {type: "SYMBOL", value: "b"}
+ * ]
+ * ```
+ */
+export function tokenize(expr: string): Token[] {
+  const result: Token[] = [];
+  let token: boolean | Token = true;
+
+  while (token) {
+    expr = expr.trim();
+    if (expr) {
+      for (let tokenizer of TOKENIZERS) {
+        token = tokenizer(expr);
+        if (token) {
+          result.push(token);
+          expr = expr.slice(token.size || token.value.length);
+          break;
+        }
+      }
+    } else {
+      token = false;
+    }
+  }
+  if (expr.length) {
+    throw new Error("Tokenizer error...");
+  }
+  return result;
+}
+
+//------------------------------------------------------------------------------
+// Expression "evaluator"
+//------------------------------------------------------------------------------
+
+/**
+ * This is the main function exported by this file. This is the code that will
+ * process an expression (given as a string) and returns another expression with
+ * proper lookups in the context.
+ *
+ * Usually, this kind of code would be very simple to do if we had an AST (so,
+ * if we had a javascript parser), since then, we would only need to find the
+ * variables and replace them.  However, a parser is more complicated, and there
+ * are no standard builtin parser API.
+ *
+ * Since this method is applied to simple javasript expressions, and the work to
+ * be done is actually quite simple, we actually can get away with not using a
+ * parser, which helps with the code size.
+ *
+ * Here is the heuristic used by this method to determine if a token is a
+ * variable:
+ * - by default, all symbols are considered a variable
+ * - unless the previous token is a dot (in that case, this is a property: `a.b`)
+ * - or if the previous token is a left brace or a comma, and the next token is
+ *   a colon (in that case, this is an object key: `{a: b}`)
+ */
+export function compileExpr(
+  expr: string,
+  vars: { [key: string]: QWebVar }
+): string {
+  const tokens = tokenize(expr);
+  let result = "";
+  for (let i = 0; i < tokens.length; i++) {
+    let token = tokens[i];
+    if (token.type === "SYMBOL" && !RESERVED_WORDS.includes(token.value)) {
+      // we need to find if it is a variable
+      let isVar = true;
+      let prevToken = tokens[i - 1];
+      if (prevToken) {
+        if (prevToken.type === "OPERATOR" && prevToken.value === ".") {
+          isVar = false;
+        } else if (
+          prevToken.type === "LEFT_BRACE" ||
+          prevToken.type === "COMMA"
+        ) {
+          let nextToken = tokens[i + 1];
+          if (nextToken && nextToken.type === "COLON") {
+            isVar = false;
+          }
+        }
+      }
+      if (isVar) {
+        if (token.value in vars && "id" in vars[token.value]) {
+          token.value = (<QWebExprVar>vars[token.value]).id;
+        } else {
+          token.value = `context['${token.value}']`;
+        }
+      }
+    }
+    result += token.value;
+  }
+  return result;
+}

tests/qweb.test.ts

@@ -84,23 +84,6 @@ describe("error handling", () => {
     ).toThrow("Missing event name with t-on directive");
   });
 
-  test("error when compiled code is invalid", () => {
-    const consoleWarn = console.warn;
-    const consoleGroupCollapsed = console.groupCollapsed;
-    console.warn = jest.fn();
-    console.groupCollapsed = jest.fn();
-    qweb.addTemplate(
-      "templatename",
-      `<div t-att-hey="}/^function invalid{{>'"></div>`
-    );
-    expect(() => qweb.render("templatename")).toThrow(
-      "Invalid generated code while compiling template 'templatename': Unexpected token }"
-    );
-    expect(console.warn).toBeCalledTimes(1);
-    console.warn = consoleWarn;
-    console.groupCollapsed = consoleGroupCollapsed;
-  });
-
   test("error when unknown directive", () => {
     qweb.addTemplate(
       "templatename",
@@ -530,7 +513,10 @@ describe("attributes", () => {
   });
 
   test("class and t-attf-class with ternary operation", () => {
-    qweb.addTemplate("test", `<div class="hello" t-attf-class="{{value ? 'world' : ''}}"/>`);
+    qweb.addTemplate(
+      "test",
+      `<div class="hello" t-attf-class="{{value ? 'world' : ''}}"/>`
+    );
     const result = renderToString(qweb, "test", { value: true });
     expect(result).toBe(`<div class="hello world"></div>`);
   });

tests/qweb_expressions.test.ts

@@ -0,0 +1,170 @@
+import { compileExpr, tokenize } from "../src/qweb_expressions";
+
+describe("tokenizer", () => {
+  test("simple tokens", () => {
+    expect(tokenize("1.3")).toEqual([
+      { type: "VALUE", value: "1.3" }
+    ]);
+
+    expect(tokenize("{}")).toEqual([
+      { type: "LEFT_BRACE", value: "{" },
+      { type: "RIGHT_BRACE", value: "}" }
+    ]);
+    expect(tokenize("{ }}")).toEqual([
+      { type: "LEFT_BRACE", value: "{" },
+      { type: "RIGHT_BRACE", value: "}" },
+      { type: "RIGHT_BRACE", value: "}" }
+    ]);
+    expect(tokenize("a")).toEqual([{ type: "SYMBOL", value: "a" }]);
+    expect(tokenize("true")).toEqual([{ type: "SYMBOL", value: "true" }]);
+    expect(tokenize("abcde")).toEqual([{ type: "SYMBOL", value: "abcde" }]);
+    expect(tokenize("_ab2")).toEqual([{ type: "SYMBOL", value: "_ab2" }]);
+    expect(tokenize("$ab2")).toEqual([{ type: "SYMBOL", value: "$ab2" }]);
+    expect(tokenize("ABC")).toEqual([{ type: "SYMBOL", value: "ABC" }]);
+
+    expect(tokenize("{a: 2}")).toEqual([
+      { type: "LEFT_BRACE", value: "{" },
+      { type: "SYMBOL", value: "a" },
+      { type: "COLON", value: ":" },
+      { type: "VALUE", value: "2" },
+      { type: "RIGHT_BRACE", value: "}" }
+    ]);
+    expect(tokenize("a,")).toEqual([
+      { type: "SYMBOL", value: "a" },
+      { type: "COMMA", value: "," }
+    ]);
+    expect(tokenize("][")).toEqual([
+      { type: "RIGHT_BRACKET", value: "]" },
+      { type: "LEFT_BRACKET", value: "[" }
+    ]);
+  });
+
+  test("strings", () => {
+    expect(() => tokenize("'")).toThrow("Invalid expression");
+    expect(() => tokenize("'\\")).toThrow("Invalid expression");
+    expect(() => tokenize("'\\'")).toThrow("Invalid expression");
+    expect(tokenize("'hello ged'")).toEqual([
+      { type: "VALUE", value: "'hello ged'" }
+    ]);
+    expect(tokenize("'hello \\'ged\\''")).toEqual([
+      { type: "VALUE", value: "'hello \\'ged\\''" }
+    ]);
+
+    expect(() => tokenize('"')).toThrow("Invalid expression");
+    expect(() => tokenize('"\\"')).toThrow("Invalid expression");
+    expect(tokenize('"hello ged"')).toEqual([
+      { type: "VALUE", value: '"hello ged"' }
+    ]);
+    expect(tokenize('"hello ged"}')).toEqual([
+      { type: "VALUE", value: '"hello ged"' },
+      { type: "RIGHT_BRACE", value: "}" }
+    ]);
+    expect(tokenize('"hello \\"ged\\""')).toEqual([
+      { type: "VALUE", value: '"hello \\"ged\\""' }
+    ]);
+  });
+});
+
+describe("expression evaluation", () => {
+  test("simple static values", () => {
+    expect(compileExpr("1", {})).toBe("1");
+    expect(compileExpr("1 ", {})).toBe("1");
+    expect(compileExpr("'some string#/, {' ", {})).toBe("'some string#/, {'");
+    expect(compileExpr("{ } ", {})).toBe("{}");
+    expect(compileExpr("{a: 1} ", {})).toBe("{a:1}");
+    expect(compileExpr("{a: 1, b: 2   } ", {})).toBe("{a:1,b:2}");
+    expect(compileExpr("[] ", {})).toBe("[]");
+    expect(compileExpr("[1] ", {})).toBe("[1]");
+    expect(compileExpr("['1', '2'] ", {})).toBe("['1','2']");
+    expect(compileExpr("['1', \"2\"] ", {})).toBe("['1',\"2\"]");
+  });
+
+  test("various types of 'words'", () => {
+    expect(compileExpr("true", {})).toBe("true");
+    expect(compileExpr("false", {})).toBe("false");
+    expect(compileExpr("debugger", {})).toBe("debugger");
+  });
+
+  test("parenthesis", () => {
+    expect(compileExpr("(1)", {})).toBe("(1)");
+    expect(compileExpr("a*(1 +3)", {})).toBe("context['a']*(1+3)");
+  });
+
+  test("objects and sub objects", () => {
+    expect(compileExpr("{a:{b:1}} ", {})).toBe("{a:{b:1}}");
+  });
+
+  test("replacing variables", () => {
+    expect(compileExpr("a", {})).toBe("context['a']");
+    expect(compileExpr("a", { a: { id: "_3", expr: "" } })).toBe("_3");
+  });
+
+  test("arrays and objects", () => {
+    expect(compileExpr("[{b:1}] ", {})).toBe("[{b:1}]");
+    expect(compileExpr("{a: []} ", {})).toBe("{a:[]}");
+    expect(compileExpr("[{b:1, c: [1, {d: {e: 3}} ]}] ", {})).toBe(
+      "[{b:1,c:[1,{d:{e:3}}]}]"
+    );
+  });
+
+  test("dot operator", () => {
+    expect(compileExpr("a.b", {})).toBe("context['a'].b");
+    expect(compileExpr("a.b.c", {})).toBe("context['a'].b.c");
+  });
+
+  test("various unary operators", () => {
+    expect(compileExpr("!flag", {})).toBe("!context['flag']");
+    expect(compileExpr("-3", {})).toBe("-3");
+    expect(compileExpr("-a", {})).toBe("-context['a']");
+  });
+
+  test("various binary operators", () => {
+    expect(compileExpr("color == 'black'", {})).toBe(
+      "context['color']=='black'"
+    );
+    expect(compileExpr("a || b", {})).toBe("context['a']||context['b']");
+    expect(compileExpr("color === 'black'", {})).toBe(
+      "context['color']==='black'"
+    );
+    expect(compileExpr("'li_'+item", {})).toBe("'li_'+context['item']");
+    expect(compileExpr("state.val > 1", {})).toBe("context['state'].val>1");
+  });
+
+  test("boolean operations", () => {
+    expect(compileExpr("a && b", {})).toBe("context['a']&&context['b']");
+  });
+
+  test("ternary operators", () => {
+    expect(compileExpr("a ? b: '2'", {})).toBe("context['a']?context['b']:'2'");
+    expect(compileExpr("a ? b: (c or '2') ", {})).toBe(
+      "context['a']?context['b']:(context['c']||'2')"
+    );
+    expect(compileExpr("a ? {test:c}: [1,u]", {})).toBe(
+      "context['a']?{test:context['c']}:[1,context['u']]"
+    );
+  });
+
+  test("word replacement", () => {
+    expect(compileExpr("a or b", {})).toBe("context['a']||context['b']");
+    expect(compileExpr("a and b", {})).toBe("context['a']&&context['b']");
+  });
+
+  test("function calls", () => {
+    expect(compileExpr("a()", {})).toBe("context['a']()");
+    expect(compileExpr("a(1)", {})).toBe("context['a'](1)");
+    expect(compileExpr("a(1,2)", {})).toBe("context['a'](1,2)");
+    expect(compileExpr("a(1,2,{a:[a]})", {})).toBe(
+      "context['a'](1,2,{a:[context['a']]})"
+    );
+    expect(compileExpr("'x'.toUpperCase()", {})).toBe("'x'.toUpperCase()");
+    expect(compileExpr("'x'.toUpperCase({a: 3})", {})).toBe(
+      "'x'.toUpperCase({a:3})"
+    );
+    expect(
+      compileExpr("'x'.toUpperCase(a)", { a: { id: "_v5", expr: "" } })
+    ).toBe("'x'.toUpperCase(_v5)");
+    expect(
+      compileExpr("'x'.toUpperCase({b: a})", { a: { id: "_v5", expr: "" } })
+    ).toBe("'x'.toUpperCase({b:_v5})");
+  });
+});