[EXT] pyjsdoc, pyjsparser

_extensions/pyjsdoc/LICENSE

@@ -0,0 +1,174 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.

_extensions/pyjsdoc/__init__.py

@@ -0,0 +1,408 @@
+from __future__ import division
+import re, json
+
+##### Parsing utilities #####
+
+def split_delimited(delimiters, split_by, text):
+    """
+    Generator that walks the ``text`` and splits it into an array on
+    ``split_by``, being careful not to break inside a delimiter pair.
+    ``delimiters`` should be an even-length string with each pair of matching
+    delimiters listed together, open first.
+
+
+    >>> list(split_delimited('{}[]', ',', ''))
+    ['']
+    >>> list(split_delimited('', ',', 'foo,bar'))
+    ['foo', 'bar']
+    >>> list(split_delimited('[]', ',', 'foo,[bar, baz]'))
+    ['foo', '[bar, baz]']
+    >>> list(split_delimited('{}', ' ', '{Type Name} name Desc'))
+    ['{Type Name}', 'name', 'Desc']
+    >>> list(split_delimited('[]{}', ',', '[{foo,[bar, baz]}]'))
+    ['[{foo,[bar, baz]}]']
+
+    Two adjacent delimiters result in a zero-length string between them:
+
+    >>> list(split_delimited('{}', ' ', '{Type Name}  Desc'))
+    ['{Type Name}', '', 'Desc']
+
+    ``split_by`` may be a predicate function instead of a string, in which
+    case it should return true on a character to split.
+
+    >>> list(split_delimited('', lambda c: c in '[]{}, ', '[{foo,[bar, baz]}]'))
+    ['', '', 'foo', '', 'bar', '', 'baz', '', '', '']
+
+    """
+    delims = [0] * (len(delimiters) // 2)
+    actions = {}
+    for i in range(0, len(delimiters), 2):
+        actions[delimiters[i]] = (i // 2, 1)
+        actions[delimiters[i + 1]] = (i // 2, -1)
+
+    if isinstance(split_by, str):
+        def split_fn(c):
+            return c == split_by
+    else:
+        split_fn = split_by
+    last = 0
+
+    for i in range(len(text)):
+        c = text[i]
+        if split_fn(c) and not any(delims):
+            yield text[last:i]
+            last = i + 1
+        try:
+            which, dir = actions[c]
+            delims[which] = delims[which] + dir
+        except KeyError:
+            pass  # Normal character
+    yield text[last:]
+
+
+def strip_stars(doc_comment):
+    r"""
+    Strip leading stars from a doc comment.
+
+    >>> strip_stars('/** This is a comment. */')
+    'This is a comment.'
+    >>> strip_stars('/**\n * This is a\n * multiline comment. */')
+    'This is a\n multiline comment.'
+    >>> strip_stars('/** \n\t * This is a\n\t * multiline comment. \n*/')
+    'This is a\n multiline comment.'
+
+    """
+    return re.sub('\n\s*?\*\s*?', '\n', doc_comment[3:-2]).strip()
+
+
+def split_tag(section):
+    """
+    Split the JSDoc tag text (everything following the @) at the first
+    whitespace.  Returns a tuple of (tagname, body).
+    """
+    splitval = re.split('\s+', section, 1)
+    tag, body = len(splitval) > 1 and splitval or (splitval[0], '')
+    return tag.strip(), body.strip()
+
+
+FUNCTION_REGEXPS = [
+    'function (\w+)',
+    '(\w+):\sfunction',
+    '\.(\w+)\s*=\s*function',
+]
+
+
+def guess_function_name(next_line, regexps=FUNCTION_REGEXPS):
+    """
+    Attempt to determine the function name from the first code line
+    following the comment.  The patterns recognized are described by
+    `regexps`, which defaults to FUNCTION_REGEXPS.  If a match is successful,
+    returns the function name.  Otherwise, returns None.
+    """
+    for regexp in regexps:
+        match = re.search(regexp, next_line)
+        if match:
+            return match.group(1)
+    return None
+
+
+def guess_parameters(next_line):
+    """
+    Attempt to guess parameters based on the presence of a parenthesized
+    group of identifiers.  If successful, returns a list of parameter names;
+    otherwise, returns None.
+    """
+    match = re.search('\(([\w\s,]+)\)', next_line)
+    if match:
+        return [arg.strip() for arg in match.group(1).split(',')]
+    else:
+        return None
+
+
+def parse_comment(doc_comment, next_line):
+    r"""
+    Split the raw comment text into a dictionary of tags.  The main comment
+    body is included as 'doc'.
+
+    >>> comment = get_doc_comments(read_file('examples/module.js'))[4][0]
+    >>> parse_comment(strip_stars(comment), '')['doc']
+    'This is the documentation for the fourth function.\n\n Since the function being documented is itself generated from another\n function, its name needs to be specified explicitly. using the @function tag'
+    >>> parse_comment(strip_stars(comment), '')['function']
+    'not_auto_discovered'
+
+    If there are multiple tags with the same name, they're included as a list:
+
+    >>> parse_comment(strip_stars(comment), '')['param']
+    ['{String} arg1 The first argument.', '{Int} arg2 The second argument.']
+
+    """
+    sections = re.split('\n\s*@', doc_comment)
+    tags = {
+        'doc': sections[0].strip(),
+        'guessed_function': guess_function_name(next_line),
+        'guessed_params': guess_parameters(next_line)
+    }
+    for section in sections[1:]:
+        tag, body = split_tag(section)
+        if tag in tags:
+            existing = tags[tag]
+            try:
+                existing.append(body)
+            except AttributeError:
+                tags[tag] = [existing, body]
+        else:
+            tags[tag] = body
+    return tags
+
+
+#### Classes #####
+
+class CommentDoc(object):
+    """
+    Base class for all classes that represent a parsed comment of some sort.
+    """
+
+    def __init__(self, parsed_comment):
+        self.parsed = parsed_comment
+
+    def __str__(self):
+        return "Docs for " + self.name
+
+    def __repr__(self):
+        return str(self)
+
+    def __contains__(self, tag_name):
+        return tag_name in self.parsed
+
+    def __getitem__(self, tag_name):
+        return self.get(tag_name)
+
+    def get(self, tag_name, default=''):
+        """
+        Return the value of a particular tag, or None if that tag doesn't
+        exist.  Use 'doc' for the comment body itself.
+        """
+        return self.parsed.get(tag_name, default)
+
+    def get_as_list(self, tag_name):
+        """
+        Return the value of a tag, making sure that it's a list.  Absent
+        tags are returned as an empty-list; single tags are returned as a
+        one-element list.
+
+        The returned list is a copy, and modifications do not affect the
+        original object.
+        """
+        val = self.get(tag_name, [])
+        if isinstance(val, list):
+            return val[:]
+        else:
+            return [val]
+
+    @property
+    def doc(self):
+        """
+        Return the comment body.
+        """
+        return self.get('doc')
+
+    @property
+    def url(self):
+        """
+        Return a URL for the comment, within the page.
+        """
+        return '#' + self.name
+
+    @property
+    def see(self):
+        """
+        Return a list of all @see tags on the comment.
+        """
+        return self.get_as_list('see')
+
+    def to_json(self):
+        """
+        Return a JSON representation of the CommentDoc.  Keys are as per
+        to_dict.
+        """
+        return json.dumps(self.to_dict())
+
+    def to_dict(self):
+        """
+        Return a dictionary representation of the CommentDoc.  The keys of
+        this correspond to the tags in the comment, with the comment body in
+        `doc`.
+        """
+        return self.parsed.copy()
+
+class ParamDoc(object):
+    """
+    Represents a parameter, option, or parameter-like object, basically
+    anything that has a name, a type, and a description, separated by spaces.
+    This is also used for return types and exceptions, which use an empty
+    string for the name.
+
+    >>> param = ParamDoc('{Array<DOM>} elems The elements to act upon')
+    >>> param.name
+    'elems'
+    >>> param.doc
+    'The elements to act upon'
+    >>> param.type
+    'Array<DOM>'
+
+    You can also omit the type: if the first element is not surrounded by
+    curly braces, it's assumed to be the name instead:
+
+    >>> param2 = ParamDoc('param1 The first param')
+    >>> param2.type
+    ''
+    >>> param2.name
+    'param1'
+    >>> param2.doc
+    'The first param'
+
+    """
+
+    def __init__(self, text):
+        parsed = list(split_delimited('{}', ' ', text))
+        if parsed[0].startswith('{') and parsed[0].endswith('}'):
+            self.type = parsed[0][1:-1]
+            self.name = parsed[1]
+            self.doc = ' '.join(parsed[2:])
+        else:
+            self.type = ''
+            self.name = parsed[0]
+            self.doc = ' '.join(parsed[1:])
+
+    def to_dict(self):
+        """
+        Convert this to a dict.  Keys (all strings) are:
+
+            - **name**: Parameter name
+            - **type**: Parameter type
+            - **doc**: Parameter description
+        """
+        return {
+            'name': self.name,
+            'type': self.type,
+            'doc': self.doc
+        }
+
+    def to_html(self, css_class=''):
+        """
+        Returns the parameter as a dt/dd pair.
+        """
+        if self.name and self.type:
+            header_text = '%s (%s)' % (self.name, self.type)
+        elif self.type:
+            header_text = self.type
+        else:
+            header_text = self.name
+        return '<dt>%s</dt><dd>%s</dd>' % (header_text, self.doc)
+
+
+##### DEPENDENCIES #####
+
+class CyclicDependency(Exception):
+    """
+    Exception raised if there is a cyclic dependency.
+    """
+
+    def __init__(self, remaining_dependencies):
+        self.values = remaining_dependencies
+
+    def __str__(self):
+        return ('The following dependencies result in a cycle: '
+                + ', '.join(self.values))
+
+
+class MissingDependency(Exception):
+    """
+    Exception raised if a file references a dependency that doesn't exist.
+    """
+
+    def __init__(self, file, dependency):
+        self.file = file
+        self.dependency = dependency
+
+    def __str__(self):
+        return "Couldn't find dependency %s when processing %s" % \
+               (self.dependency, self.file)
+
+
+def build_dependency_graph(start_nodes, js_doc):
+    """
+    Build a graph where nodes are filenames and edges are reverse dependencies
+    (so an edge from jquery.js to jquery.dimensions.js indicates that jquery.js
+    must be included before jquery.dimensions.js).  The graph is represented
+    as a dictionary from filename to (in-degree, edges) pair, for ease of
+    topological sorting.  Also returns a list of nodes of degree zero.
+    """
+    queue = []
+    dependencies = {}
+    start_sort = []
+
+    def add_vertex(file):
+        in_degree = len(js_doc[file].module.dependencies)
+        dependencies[file] = [in_degree, []]
+        queue.append(file)
+        if in_degree == 0:
+            start_sort.append(file)
+
+    def add_edge(from_file, to_file):
+        dependencies[from_file][1].append(to_file)
+
+    def is_in_graph(file):
+        return file in dependencies
+
+    for file in start_nodes:
+        add_vertex(file)
+    for file in queue:
+        for dependency in js_doc[file].module.dependencies:
+            if dependency not in js_doc:
+                raise MissingDependency(file, dependency)
+            if not is_in_graph(dependency):
+                add_vertex(dependency)
+            add_edge(dependency, file)
+    return dependencies, start_sort
+
+
+def topological_sort(dependencies, start_nodes):
+    """
+    Perform a topological sort on the dependency graph `dependencies`, starting
+    from list `start_nodes`.
+    """
+    retval = []
+
+    def edges(node):
+        return dependencies[node][1]
+
+    def in_degree(node):
+        return dependencies[node][0]
+
+    def remove_incoming(node):
+        dependencies[node][0] = in_degree(node) - 1
+
+    while start_nodes:
+        node = start_nodes.pop()
+        retval.append(node)
+        for child in edges(node):
+            remove_incoming(child)
+            if not in_degree(child):
+                start_nodes.append(child)
+    leftover_nodes = [node for node in dependencies.keys()
+                      if in_degree(node) > 0]
+    if leftover_nodes:
+        raise CyclicDependency(leftover_nodes)
+    else:
+        return retval
+
+
+def find_dependencies(start_nodes, js_doc):
+    """
+    Sort the dependency graph, taking in a list of starting module names and a
+    CodeBaseDoc (or equivalent dictionary).  Returns an ordered list of
+    transitive dependencies such that no module appears before its
+    dependencies.
+    """
+    return topological_sort(*build_dependency_graph(start_nodes, js_doc))

_extensions/pyjsparser/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2017 Piotr Dabkowski
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

_extensions/pyjsparser/README

@@ -0,0 +1,4 @@
+Vendoring of pyjsparser to add comments support as we need them for autojsdoc.
+
+Alternative: use https://github.com/Kronuz/esprima-python instead, seems to
+support comments though may be attaching them differently.

_extensions/pyjsparser/__init__.py

@@ -0,0 +1,4 @@
+s__all__ = ['PyJsParser', 'parse', 'JsSyntaxError']
+__author__ = 'Piotr Dabkowski'
+__version__ = '2.2.0'
+from .parser import PyJsParser, parse, JsSyntaxError

_extensions/pyjsparser/pyjsparserdata.py

@@ -0,0 +1,297 @@
+# The MIT License
+#
+# Copyright 2014, 2015 Piotr Dabkowski
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the 'Software'),
+# to deal in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+# the Software, and to permit persons to whom the Software is furnished to do so, subject
+# to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all copies or
+# substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+# LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE
+#  OR THE USE OR OTHER DEALINGS IN THE SOFTWARE
+from __future__ import unicode_literals
+
+import sys
+import unicodedata
+from collections import defaultdict
+
+
+token = {
+        'BooleanLiteral': 1,
+        'EOF': 2,
+        'Identifier': 3,
+        'Keyword': 4,
+        'NullLiteral': 5,
+        'NumericLiteral': 6,
+        'Punctuator': 7,
+        'StringLiteral': 8,
+        'RegularExpression': 9,
+        'Template': 10
+    }
+
+
+TokenName = dict((v,k) for k,v in token.items())
+
+FnExprTokens = ['(', '{', '[', 'in', 'typeof', 'instanceof', 'new',
+                    'return', 'case', 'delete', 'throw', 'void',
+                    # assignment operators
+                    '=', '+=', '-=', '*=', '/=', '%=', '<<=', '>>=', '>>>=',
+                    '&=', '|=', '^=', ',',
+                    # binary/unary operators
+                    '+', '-', '*', '/', '%', '++', '--', '<<', '>>', '>>>', '&',
+                    '|', '^', '!', '~', '&&', '||', '?', ':', '===', '==', '>=',
+                    '<=', '<', '>', '!=', '!==']
+
+syntax= set(('AssignmentExpression',
+         'AssignmentPattern',
+         'ArrayExpression',
+         'ArrayPattern',
+         'ArrowFunctionExpression',
+         'BlockStatement',
+         'BinaryExpression',
+         'BreakStatement',
+         'CallExpression',
+         'CatchClause',
+         'ClassBody',
+         'ClassDeclaration',
+         'ClassExpression',
+         'ConditionalExpression',
+         'ContinueStatement',
+         'DoWhileStatement',
+         'DebuggerStatement',
+         'EmptyStatement',
+         'ExportAllDeclaration',
+         'ExportDefaultDeclaration',
+         'ExportNamedDeclaration',
+         'ExportSpecifier',
+         'ExpressionStatement',
+         'ForStatement',
+         'ForInStatement',
+         'FunctionDeclaration',
+         'FunctionExpression',
+         'Identifier',
+         'IfStatement',
+         'ImportDeclaration',
+         'ImportDefaultSpecifier',
+         'ImportNamespaceSpecifier',
+         'ImportSpecifier',
+         'Literal',
+         'LabeledStatement',
+         'LogicalExpression',
+         'MemberExpression',
+         'MethodDefinition',
+         'NewExpression',
+         'ObjectExpression',
+         'ObjectPattern',
+         'Program',
+         'Property',
+         'RestElement',
+         'ReturnStatement',
+         'SequenceExpression',
+         'SpreadElement',
+         'Super',
+         'SwitchCase',
+         'SwitchStatement',
+         'TaggedTemplateExpression',
+         'TemplateElement',
+         'TemplateLiteral',
+         'ThisExpression',
+         'ThrowStatement',
+         'TryStatement',
+         'UnaryExpression',
+         'UpdateExpression',
+         'VariableDeclaration',
+         'VariableDeclarator',
+         'WhileStatement',
+         'WithStatement'))
+
+
+# Error messages should be identical to V8.
+messages = {
+        'UnexpectedToken': 'Unexpected token %s',
+        'UnexpectedNumber': 'Unexpected number',
+        'UnexpectedString': 'Unexpected string',
+        'UnexpectedIdentifier': 'Unexpected identifier',
+        'UnexpectedReserved': 'Unexpected reserved word',
+        'UnexpectedTemplate': 'Unexpected quasi %s',
+        'UnexpectedEOS': 'Unexpected end of input',
+        'NewlineAfterThrow': 'Illegal newline after throw',
+        'InvalidRegExp': 'Invalid regular expression',
+        'UnterminatedRegExp': 'Invalid regular expression: missing /',
+        'InvalidLHSInAssignment': 'Invalid left-hand side in assignment',
+        'InvalidLHSInForIn': 'Invalid left-hand side in for-in',
+        'MultipleDefaultsInSwitch': 'More than one default clause in switch statement',
+        'NoCatchOrFinally': 'Missing catch or finally after try',
+        'UnknownLabel': 'Undefined label \'%s\'',
+        'Redeclaration': '%s \'%s\' has already been declared',
+        'IllegalContinue': 'Illegal continue statement',
+        'IllegalBreak': 'Illegal break statement',
+        'IllegalReturn': 'Illegal return statement',
+        'StrictModeWith': 'Strict mode code may not include a with statement',
+        'StrictCatchVariable': 'Catch variable may not be eval or arguments in strict mode',
+        'StrictVarName': 'Variable name may not be eval or arguments in strict mode',
+        'StrictParamName': 'Parameter name eval or arguments is not allowed in strict mode',
+        'StrictParamDupe': 'Strict mode function may not have duplicate parameter names',
+        'StrictFunctionName': 'Function name may not be eval or arguments in strict mode',
+        'StrictOctalLiteral': 'Octal literals are not allowed in strict mode.',
+        'StrictDelete': 'Delete of an unqualified identifier in strict mode.',
+        'StrictLHSAssignment': 'Assignment to eval or arguments is not allowed in strict mode',
+        'StrictLHSPostfix': 'Postfix increment/decrement may not have eval or arguments operand in strict mode',
+        'StrictLHSPrefix': 'Prefix increment/decrement may not have eval or arguments operand in strict mode',
+        'StrictReservedWord': 'Use of future reserved word in strict mode',
+        'TemplateOctalLiteral': 'Octal literals are not allowed in template strings.',
+        'ParameterAfterRestParameter': 'Rest parameter must be last formal parameter',
+        'DefaultRestParameter': 'Unexpected token =',
+        'ObjectPatternAsRestParameter': 'Unexpected token {',
+        'DuplicateProtoProperty': 'Duplicate __proto__ fields are not allowed in object literals',
+        'ConstructorSpecialMethod': 'Class constructor may not be an accessor',
+        'DuplicateConstructor': 'A class may only have one constructor',
+        'StaticPrototype': 'Classes may not have static property named prototype',
+        'MissingFromClause': 'Unexpected token',
+        'NoAsAfterImportNamespace': 'Unexpected token',
+        'InvalidModuleSpecifier': 'Unexpected token',
+        'IllegalImportDeclaration': 'Unexpected token',
+        'IllegalExportDeclaration': 'Unexpected token'}
+
+PRECEDENCE = {'||':1,
+             '&&':2,
+             '|':3,
+             '^':4,
+             '&':5,
+             '==':6,
+             '!=':6,
+             '===':6,
+             '!==':6,
+             '<':7,
+             '>':7,
+             '<=':7,
+             '>=':7,
+             'instanceof':7,
+             'in':7,
+             '<<':8,
+             '>>':8,
+             '>>>':8,
+             '+':9,
+             '-':9,
+             '*':11,
+             '/':11,
+             '%':11}
+
+class Token: pass
+class Syntax: pass
+class Messages: pass
+class PlaceHolders:
+    ArrowParameterPlaceHolder = 'ArrowParameterPlaceHolder'
+
+for k,v in token.items():
+    setattr(Token, k, v)
+
+for e in syntax:
+    setattr(Syntax, e, e)
+
+for k,v in messages.items():
+    setattr(Messages, k, v)
+
+#http://stackoverflow.com/questions/14245893/efficiently-list-all-characters-in-a-given-unicode-category
+BOM = u'\uFEFF'
+ZWJ = u'\u200D'
+ZWNJ = u'\u200C'
+TAB = u'\u0009'
+VT = u'\u000B'
+FF = u'\u000C'
+SP = u'\u0020'
+NBSP = u'\u00A0'
+LF = u'\u000A'
+CR = u'\u000D'
+LS = u'\u2028'
+PS = u'\u2029'
+
+U_CATEGORIES = defaultdict(list)
+for c in map(chr, range(sys.maxunicode + 1)):
+    U_CATEGORIES[unicodedata.category(c)].append(c)
+UNICODE_LETTER = set(U_CATEGORIES['Lu']+U_CATEGORIES['Ll']+
+                     U_CATEGORIES['Lt']+U_CATEGORIES['Lm']+
+                     U_CATEGORIES['Lo']+U_CATEGORIES['Nl'])
+UNICODE_COMBINING_MARK = set(U_CATEGORIES['Mn']+U_CATEGORIES['Mc'])
+UNICODE_DIGIT = set(U_CATEGORIES['Nd'])
+UNICODE_CONNECTOR_PUNCTUATION = set(U_CATEGORIES['Pc'])
+IDENTIFIER_START = UNICODE_LETTER.union(set(('$','_', '\\'))) # and some fucking unicode escape sequence
+IDENTIFIER_PART = IDENTIFIER_START.union(UNICODE_COMBINING_MARK).union(UNICODE_DIGIT)\
+    .union(UNICODE_CONNECTOR_PUNCTUATION).union(set((ZWJ, ZWNJ)))
+
+WHITE_SPACE = set((0x20, 0x09, 0x0B, 0x0C, 0xA0, 0x1680,
+               0x180E, 0x2000, 0x2001, 0x2002, 0x2003,
+                0x2004, 0x2005, 0x2006, 0x2007, 0x2008,
+                0x2009, 0x200A, 0x202F, 0x205F, 0x3000,
+                0xFEFF))
+
+LINE_TERMINATORS = set((0x0A, 0x0D, 0x2028, 0x2029))
+
+def isIdentifierStart(ch):
+    return (ch if isinstance(ch, str) else chr(ch))  in IDENTIFIER_START
+
+def isIdentifierPart(ch):
+    return (ch if isinstance(ch, str) else chr(ch))  in IDENTIFIER_PART
+
+def isWhiteSpace(ch):
+    return (ord(ch) if isinstance(ch, str) else ch) in WHITE_SPACE
+
+def isLineTerminator(ch):
+    return (ord(ch) if isinstance(ch, str) else ch)  in LINE_TERMINATORS
+
+OCTAL = set(('0', '1', '2', '3', '4', '5', '6', '7'))
+DEC = set(('0', '1', '2', '3', '4', '5', '6', '7', '8', '9'))
+HEX = set('0123456789abcdefABCDEF')
+HEX_CONV = dict(('0123456789abcdef'[n],n) for n in range(16))
+for i,e in enumerate('ABCDEF', 10):
+    HEX_CONV[e] = i
+
+
+def isDecimalDigit(ch):
+    return (ch if isinstance(ch, str) else chr(ch)) in DEC
+
+def isHexDigit(ch):
+    return (ch if isinstance(ch, str) else chr(ch))  in HEX
+
+def isOctalDigit(ch):
+    return (ch if isinstance(ch, str) else chr(ch))  in OCTAL
+
+def isFutureReservedWord(w):
+    return w in ('enum', 'export', 'import', 'super')
+
+
+RESERVED_WORD = set(('implements', 'interface', 'package', 'private', 'protected', 'public', 'static', 'yield', 'let'))
+def isStrictModeReservedWord(w):
+    return w in RESERVED_WORD
+
+def isRestrictedWord(w):
+    return w in  ('eval', 'arguments')
+
+
+KEYWORDS = set(('if', 'in', 'do', 'var', 'for', 'new', 'try', 'let', 'this', 'else', 'case',
+                     'void', 'with', 'enum', 'while', 'break', 'catch', 'throw', 'const', 'yield',
+                     'class', 'super', 'return', 'typeof', 'delete', 'switch', 'export', 'import',
+                     'default', 'finally', 'extends', 'function', 'continue', 'debugger', 'instanceof', 'pyimport'))
+def isKeyword(w):
+        # 'const' is specialized as Keyword in V8.
+        # 'yield' and 'let' are for compatibility with SpiderMonkey and ES.next.
+        # Some others are from future reserved words.
+        return w in KEYWORDS
+
+
+class JsSyntaxError(Exception): pass
+
+if __name__=='__main__':
+    assert isLineTerminator('\n')
+    assert isLineTerminator(0x0A)
+    assert isIdentifierStart('$')
+    assert isIdentifierStart(100)
+    assert isWhiteSpace(' ')

_extensions/pyjsparser/std_nodes.py

@@ -0,0 +1,470 @@
+from .pyjsparserdata import *
+
+
+class BaseNode:
+    def finish(self):
+        pass
+
+    def finishArrayExpression(self, elements):
+        self.type = Syntax.ArrayExpression
+        self.elements = elements
+        self.finish()
+        return self
+
+    def finishArrayPattern(self, elements):
+        self.type = Syntax.ArrayPattern
+        self.elements = elements
+        self.finish()
+        return self
+
+    def finishArrowFunctionExpression(self, params, defaults, body, expression):
+        self.type = Syntax.ArrowFunctionExpression
+        self.id = None
+        self.params = params
+        self.defaults = defaults
+        self.body = body
+        self.generator = False
+        self.expression = expression
+        self.finish()
+        return self
+
+    def finishAssignmentExpression(self, operator, left, right):
+        self.type = Syntax.AssignmentExpression
+        self.operator = operator
+        self.left = left
+        self.right = right
+        self.finish()
+        return self
+
+    def finishAssignmentPattern(self, left, right):
+        self.type = Syntax.AssignmentPattern
+        self.left = left
+        self.right = right
+        self.finish()
+        return self
+
+    def finishBinaryExpression(self, operator, left, right):
+        self.type = Syntax.LogicalExpression if (operator == '||' or operator == '&&') else Syntax.BinaryExpression
+        self.operator = operator
+        self.left = left
+        self.right = right
+        self.finish()
+        return self
+
+    def finishBlockStatement(self, body):
+        self.type = Syntax.BlockStatement
+        self.body = body
+        self.finish()
+        return self
+
+    def finishBreakStatement(self, label):
+        self.type = Syntax.BreakStatement
+        self.label = label
+        self.finish()
+        return self
+
+    def finishCallExpression(self, callee, args):
+        self.type = Syntax.CallExpression
+        self.callee = callee
+        self.arguments = args
+        self.finish()
+        return self
+
+    def finishCatchClause(self, param, body):
+        self.type = Syntax.CatchClause
+        self.param = param
+        self.body = body
+        self.finish()
+        return self
+
+    def finishClassBody(self, body):
+        self.type = Syntax.ClassBody
+        self.body = body
+        self.finish()
+        return self
+
+    def finishClassDeclaration(self, id, superClass, body):
+        self.type = Syntax.ClassDeclaration
+        self.id = id
+        self.superClass = superClass
+        self.body = body
+        self.finish()
+        return self
+
+    def finishClassExpression(self, id, superClass, body):
+        self.type = Syntax.ClassExpression
+        self.id = id
+        self.superClass = superClass
+        self.body = body
+        self.finish()
+        return self
+
+    def finishConditionalExpression(self, test, consequent, alternate):
+        self.type = Syntax.ConditionalExpression
+        self.test = test
+        self.consequent = consequent
+        self.alternate = alternate
+        self.finish()
+        return self
+
+    def finishContinueStatement(self, label):
+        self.type = Syntax.ContinueStatement
+        self.label = label
+        self.finish()
+        return self
+
+    def finishDebuggerStatement(self, ):
+        self.type = Syntax.DebuggerStatement
+        self.finish()
+        return self
+
+    def finishDoWhileStatement(self, body, test):
+        self.type = Syntax.DoWhileStatement
+        self.body = body
+        self.test = test
+        self.finish()
+        return self
+
+    def finishEmptyStatement(self, ):
+        self.type = Syntax.EmptyStatement
+        self.finish()
+        return self
+
+    def finishExpressionStatement(self, expression):
+        self.type = Syntax.ExpressionStatement
+        self.expression = expression
+        self.finish()
+        return self
+
+    def finishForStatement(self, init, test, update, body):
+        self.type = Syntax.ForStatement
+        self.init = init
+        self.test = test
+        self.update = update
+        self.body = body
+        self.finish()
+        return self
+
+    def finishForInStatement(self, left, right, body):
+        self.type = Syntax.ForInStatement
+        self.left = left
+        self.right = right
+        self.body = body
+        self.each = False
+        self.finish()
+        return self
+
+    def finishFunctionDeclaration(self, id, params, defaults, body):
+        self.type = Syntax.FunctionDeclaration
+        self.id = id
+        self.params = params
+        self.defaults = defaults
+        self.body = body
+        self.generator = False
+        self.expression = False
+        self.finish()
+        return self
+
+    def finishFunctionExpression(self, id, params, defaults, body):
+        self.type = Syntax.FunctionExpression
+        self.id = id
+        self.params = params
+        self.defaults = defaults
+        self.body = body
+        self.generator = False
+        self.expression = False
+        self.finish()
+        return self
+
+    def finishIdentifier(self, name):
+        self.type = Syntax.Identifier
+        self.name = name
+        self.finish()
+        return self
+
+    def finishIfStatement(self, test, consequent, alternate):
+        self.type = Syntax.IfStatement
+        self.test = test
+        self.consequent = consequent
+        self.alternate = alternate
+        self.finish()
+        return self
+
+    def finishLabeledStatement(self, label, body):
+        self.type = Syntax.LabeledStatement
+        self.label = label
+        self.body = body
+        self.finish()
+        return self
+
+    def finishLiteral(self, token):
+        self.type = Syntax.Literal
+        self.value = token['value']
+        self.raw = None  # todo fix it?
+        if token.get('regex'):
+            self.regex = token['regex']
+        self.finish()
+        return self
+
+    def finishMemberExpression(self, accessor, object, property):
+        self.type = Syntax.MemberExpression
+        self.computed = accessor == '['
+        self.object = object
+        self.property = property
+        self.finish()
+        return self
+
+    def finishNewExpression(self, callee, args):
+        self.type = Syntax.NewExpression
+        self.callee = callee
+        self.arguments = args
+        self.finish()
+        return self
+
+    def finishObjectExpression(self, properties):
+        self.type = Syntax.ObjectExpression
+        self.properties = properties
+        self.finish()
+        return self
+
+    def finishObjectPattern(self, properties):
+        self.type = Syntax.ObjectPattern
+        self.properties = properties
+        self.finish()
+        return self
+
+    def finishPostfixExpression(self, operator, argument):
+        self.type = Syntax.UpdateExpression
+        self.operator = operator
+        self.argument = argument
+        self.prefix = False
+        self.finish()
+        return self
+
+    def finishProgram(self, body):
+        self.type = Syntax.Program
+        self.body = body
+        self.finish()
+        return self
+
+    def finishPyimport(self, imp):
+        self.type = 'PyimportStatement'
+        self.imp = imp
+        self.finish()
+        return self
+
+    def finishProperty(self, kind, key, computed, value, method, shorthand):
+        self.type = Syntax.Property
+        self.key = key
+        self.computed = computed
+        self.value = value
+        self.kind = kind
+        self.method = method
+        self.shorthand = shorthand
+        self.finish()
+        return self
+
+    def finishRestElement(self, argument):
+        self.type = Syntax.RestElement
+        self.argument = argument
+        self.finish()
+        return self
+
+    def finishReturnStatement(self, argument):
+        self.type = Syntax.ReturnStatement
+        self.argument = argument
+        self.finish()
+        return self
+
+    def finishSequenceExpression(self, expressions):
+        self.type = Syntax.SequenceExpression
+        self.expressions = expressions
+        self.finish()
+        return self
+
+    def finishSpreadElement(self, argument):
+        self.type = Syntax.SpreadElement
+        self.argument = argument
+        self.finish()
+        return self
+
+    def finishSwitchCase(self, test, consequent):
+        self.type = Syntax.SwitchCase
+        self.test = test
+        self.consequent = consequent
+        self.finish()
+        return self
+
+    def finishSuper(self, ):
+        self.type = Syntax.Super
+        self.finish()
+        return self
+
+    def finishSwitchStatement(self, discriminant, cases):
+        self.type = Syntax.SwitchStatement
+        self.discriminant = discriminant
+        self.cases = cases
+        self.finish()
+        return self
+
+    def finishTaggedTemplateExpression(self, tag, quasi):
+        self.type = Syntax.TaggedTemplateExpression
+        self.tag = tag
+        self.quasi = quasi
+        self.finish()
+        return self
+
+    def finishTemplateElement(self, value, tail):
+        self.type = Syntax.TemplateElement
+        self.value = value
+        self.tail = tail
+        self.finish()
+        return self
+
+    def finishTemplateLiteral(self, quasis, expressions):
+        self.type = Syntax.TemplateLiteral
+        self.quasis = quasis
+        self.expressions = expressions
+        self.finish()
+        return self
+
+    def finishThisExpression(self, ):
+        self.type = Syntax.ThisExpression
+        self.finish()
+        return self
+
+    def finishThrowStatement(self, argument):
+        self.type = Syntax.ThrowStatement
+        self.argument = argument
+        self.finish()
+        return self
+
+    def finishTryStatement(self, block, handler, finalizer):
+        self.type = Syntax.TryStatement
+        self.block = block
+        self.guardedHandlers = []
+        self.handlers = [handler] if handler else []
+        self.handler = handler
+        self.finalizer = finalizer
+        self.finish()
+        return self
+
+    def finishUnaryExpression(self, operator, argument):
+        self.type = Syntax.UpdateExpression if (operator == '++' or operator == '--') else Syntax.UnaryExpression
+        self.operator = operator
+        self.argument = argument
+        self.prefix = True
+        self.finish()
+        return self
+
+    def finishVariableDeclaration(self, declarations):
+        self.type = Syntax.VariableDeclaration
+        self.declarations = declarations
+        self.kind = 'var'
+        self.finish()
+        return self
+
+    def finishLexicalDeclaration(self, declarations, kind):
+        self.type = Syntax.VariableDeclaration
+        self.declarations = declarations
+        self.kind = kind
+        self.finish()
+        return self
+
+    def finishVariableDeclarator(self, id, init):
+        self.type = Syntax.VariableDeclarator
+        self.id = id
+        self.init = init
+        self.finish()
+        return self
+
+    def finishWhileStatement(self, test, body):
+        self.type = Syntax.WhileStatement
+        self.test = test
+        self.body = body
+        self.finish()
+        return self
+
+    def finishWithStatement(self, object, body):
+        self.type = Syntax.WithStatement
+        self.object = object
+        self.body = body
+        self.finish()
+        return self
+
+    def finishExportSpecifier(self, local, exported):
+        self.type = Syntax.ExportSpecifier
+        self.exported = exported or local
+        self.local = local
+        self.finish()
+        return self
+
+    def finishImportDefaultSpecifier(self, local):
+        self.type = Syntax.ImportDefaultSpecifier
+        self.local = local
+        self.finish()
+        return self
+
+    def finishImportNamespaceSpecifier(self, local):
+        self.type = Syntax.ImportNamespaceSpecifier
+        self.local = local
+        self.finish()
+        return self
+
+    def finishExportNamedDeclaration(self, declaration, specifiers, src):
+        self.type = Syntax.ExportNamedDeclaration
+        self.declaration = declaration
+        self.specifiers = specifiers
+        self.source = src
+        self.finish()
+        return self
+
+    def finishExportDefaultDeclaration(self, declaration):
+        self.type = Syntax.ExportDefaultDeclaration
+        self.declaration = declaration
+        self.finish()
+        return self
+
+    def finishExportAllDeclaration(self, src):
+        self.type = Syntax.ExportAllDeclaration
+        self.source = src
+        self.finish()
+        return self
+
+    def finishImportSpecifier(self, local, imported):
+        self.type = Syntax.ImportSpecifier
+        self.local = local or imported
+        self.imported = imported
+        self.finish()
+        return self
+
+    def finishImportDeclaration(self, specifiers, src):
+        self.type = Syntax.ImportDeclaration
+        self.specifiers = specifiers
+        self.source = src
+        self.finish()
+        return self
+
+    def __getitem__(self, item):
+        return getattr(self, item)
+
+    def __setitem__(self, key, value):
+        setattr(self, key, value)
+
+class Node(BaseNode):
+    pass
+
+
+class WrappingNode(BaseNode):
+    def __init__(self, startToken=None):
+        pass
+
+
+def node_to_dict(node):  # extremely important for translation speed
+    if isinstance(node, list):
+        return [node_to_dict(e) for e in node]
+    elif isinstance(node, dict):
+        return dict((k, node_to_dict(v)) for k, v in node.items())
+    elif not isinstance(node, BaseNode):
+        return node
+    return dict((k, node_to_dict(v)) for k, v in node.__dict__.items())