From fcec9a9d7f4e2d456aaf6f6b3fa4a888f752e887 Mon Sep 17 00:00:00 2001 From: Victor Feyens Date: Thu, 24 Sep 2020 16:11:55 +0200 Subject: [PATCH] [TOKEEP ?] autojsdoc extension --- _extensions/autojsdoc/README.rst | 87 +++ _extensions/autojsdoc/__init__.py | 1 + _extensions/autojsdoc/__main__.py | 154 ++++ _extensions/autojsdoc/ext/__init__.py | 13 + _extensions/autojsdoc/ext/directives.py | 719 ++++++++++++++++++ _extensions/autojsdoc/ext/extractor.py | 87 +++ _extensions/autojsdoc/parser/__init__.py | 1 + _extensions/autojsdoc/parser/jsdoc.py | 357 +++++++++ _extensions/autojsdoc/parser/parser.py | 572 ++++++++++++++ _extensions/autojsdoc/parser/tests/README.rst | 6 + _extensions/autojsdoc/parser/tests/support.py | 73 ++ .../autojsdoc/parser/tests/test_class.py | 227 ++++++ .../autojsdoc/parser/tests/test_crap.py | 75 ++ .../autojsdoc/parser/tests/test_module.py | 173 +++++ .../autojsdoc/parser/tests/test_namespace.py | 177 +++++ .../autojsdoc/parser/tests/test_params.py | 116 +++ .../autojsdoc/parser/tests/test_typespec.py | 104 +++ _extensions/autojsdoc/parser/types.py | 196 +++++ _extensions/autojsdoc/parser/utils.py | 295 +++++++ _extensions/autojsdoc/parser/visitor.py | 126 +++ 20 files changed, 3559 insertions(+) create mode 100644 _extensions/autojsdoc/README.rst create mode 100644 _extensions/autojsdoc/__init__.py create mode 100644 _extensions/autojsdoc/__main__.py create mode 100644 _extensions/autojsdoc/ext/__init__.py create mode 100644 _extensions/autojsdoc/ext/directives.py create mode 100644 _extensions/autojsdoc/ext/extractor.py create mode 100644 _extensions/autojsdoc/parser/__init__.py create mode 100644 _extensions/autojsdoc/parser/jsdoc.py create mode 100644 _extensions/autojsdoc/parser/parser.py create mode 100644 _extensions/autojsdoc/parser/tests/README.rst create mode 100644 _extensions/autojsdoc/parser/tests/support.py create mode 100644 _extensions/autojsdoc/parser/tests/test_class.py create mode 100644 _extensions/autojsdoc/parser/tests/test_crap.py create mode 100644 _extensions/autojsdoc/parser/tests/test_module.py create mode 100644 _extensions/autojsdoc/parser/tests/test_namespace.py create mode 100644 _extensions/autojsdoc/parser/tests/test_params.py create mode 100644 _extensions/autojsdoc/parser/tests/test_typespec.py create mode 100644 _extensions/autojsdoc/parser/types.py create mode 100644 _extensions/autojsdoc/parser/utils.py create mode 100644 _extensions/autojsdoc/parser/visitor.py diff --git a/_extensions/autojsdoc/README.rst b/_extensions/autojsdoc/README.rst new file mode 100644 index 000000000..ef7681778 --- /dev/null +++ b/_extensions/autojsdoc/README.rst @@ -0,0 +1,87 @@ +:orphan: + +====================================== +JSDoc parser/Sphinx extension for Odoo +====================================== + +Why? +==== + +Spent about a week trying to coerce "standard" javascript tools (jsdoc_ with +the hope of using sphinx-js_ for integration or `documentation.js`_) and +failed to ever get a sensible result: failed to get any result with the +current state of the documentation, significant changes/additions/fixes to +docstrings brought this up to "garbage output" level. + +Bug reports and mailing list posts didn't show any path to improvement on the +ES5 codebase (if we ever go whole-hog on ES6 modules and classes things could +be different, in fact most of JSDoc's current effort seem focused on +ES6/ES2015 features) but both experience and looking at the mailing lists +told me that spending more time would be wasted. + +Even more so as I was writing visitors/rewriters to generate documentation +from our existing structure, which broadly speaking is relatively strict, and +thus + +What? +===== + +If it were possible to generate JSDoc annotations from our relatively +well-defined code structures, it was obviously possible to extract documentary +information directly from it, hence this Odoo-specific package/extension +trying to do exactly that. + +This package should eventually provide: + +* a command-line interface which can be invoked via ``-m autojsdoc`` (assuming + your ``PYTHONPATH`` can find it) which should allow dumping the parsed AST + in a convenient-ish form, possibly doing searches through the AST, a + dependency graph extractor/analysis and a text dumper for the + documentation. + +* a sphinx extension (``autojsdoc.sphinx``) which can be used to integrate the + parsed JSDoc information into the Sphinx doc. + +How? +==== + +Sphinx-aside, the package relies on 3 libraries: + +* pyjsparser_, an Esprima-compliant ES5.1 parser (with bits of ES6 support), + sadly it does not support comments in its current form so I had to fork it. + Fed a javascript source file, pyjsparser_ simply generates a bunch of nested + dicts representing an Esprima ast, ast-types_ does a reasonably good job of + describing it once you understand that "bases" are basically just structural + mixins. + + Because the original does not, this package provides a ``visitor`` module + for pyjsparser_ ASTs. + +* pyjsdoc_, a one-file "port" of jsdoc, can actually do much of the JS parsing + (using string munging) but its core semantics don't fit our needs so I'm + only using it to parse the actual JSDoc content, and the ``jsdoc`` module + contains some replacement classes, extensions & monkey patches for things + `pyjsdoc`_ itself does not support, at the time of this writing: + + - a bug in FunctionDoc.return_val + - a type on FunctionDoc so it's compatible with ParamDoc + - a more reliable comments-parsing function + - a replacement ModuleDoc as the original does not materialise AMD modules + - a ClassDoc extension to support mixins + - two additional CommentDoc extensions for "namespaces" objects (bag of + attributes without any more information) and mixin objects + +* pytest_ to configure and run the test suite, which you can run by invoking + ``pytest doc/_extensions`` from the project top-level, the tests represent + both "happy path" things we want to parse and various code patterns which + tripped the happy path because e.g. they were matched and should not have, + they were not matched and should have, or they were more complex than the + happy path had expected + +.. _ast-types: _https://github.com/benjamn/ast-types/blob/master/def/core.js +.. _documentation.js: http://documentation.js.org +.. _jsdoc: http://usejsdoc.org +.. _pyjsdoc: https://github.com/nostrademons/pyjsdoc +.. _pyjsparser: https://github.com/PiotrDabkowski/pyjsparser +.. _pytest: https://pytest.org/ +.. _sphinx-js: https://sphinx-js-howto.readthedocs.io diff --git a/_extensions/autojsdoc/__init__.py b/_extensions/autojsdoc/__init__.py new file mode 100644 index 000000000..40a96afc6 --- /dev/null +++ b/_extensions/autojsdoc/__init__.py @@ -0,0 +1 @@ +# -*- coding: utf-8 -*- diff --git a/_extensions/autojsdoc/__main__.py b/_extensions/autojsdoc/__main__.py new file mode 100644 index 000000000..2ac5f3a0c --- /dev/null +++ b/_extensions/autojsdoc/__main__.py @@ -0,0 +1,154 @@ +# -*- coding: utf-8 -*- +from __future__ import print_function +import cgitb +import fnmatch +import io +import logging + +import click + +import pyjsparser +import sys + +from .parser.parser import ModuleMatcher +from .parser.visitor import Visitor, SKIP +from .parser import jsdoc + +class Printer(Visitor): + def __init__(self, level=0): + super(Printer, self).__init__() + self.level = level + + def _print(self, text): + print(' ' * self.level, text) + + def enter_generic(self, node): + self._print(node['type']) + self.level += 1 + + def exit_generic(self, node): + self.level -= 1 + + def enter_Identifier(self, node): + self._print(node['name']) + return SKIP + + def enter_Literal(self, node): + self._print(node['value']) + return SKIP + + def enter_BinaryExpression(self, node): + self._print(node['operator']) + self.level += 1 + +def visit_files(files, visitor, ctx): + for name in files: + with io.open(name) as f: + ctx.logger.info("%s", name) + try: + yield visitor().visit(pyjsparser.parse(f.read())) + except Exception as e: + if ctx.logger.isEnabledFor(logging.DEBUG): + ctx.logger.exception("while visiting %s", name) + else: + ctx.logger.error("%s while visiting %s", e, name) + +# bunch of modules various bits depend on which are not statically defined +# (or are outside the scope of the system) +ABSTRACT_MODULES = [ + jsdoc.ModuleDoc({ + 'module': 'web.web_client', + 'dependency': {'web.AbstractWebClient'}, + 'exports': jsdoc.NSDoc({ + 'name': 'web_client', + 'doc': 'instance of AbstractWebClient', + }), + }), + jsdoc.ModuleDoc({ + 'module': 'web.Tour', + 'dependency': {'web_tour.TourManager'}, + 'exports': jsdoc.NSDoc({ + 'name': 'Tour', + 'doc': 'maybe tourmanager instance?', + }), + }), + # OH FOR FUCK'S SAKE + jsdoc.ModuleDoc({ + 'module': 'summernote/summernote', + 'exports': jsdoc.NSDoc({'doc': "totally real summernote"}), + }) +] + +@click.group(context_settings={'help_option_names': ['-h', '--help']}) +@click.option('-v', '--verbose', count=True) +@click.option('-q', '--quiet', count=True) +@click.pass_context +def autojsdoc(ctx, verbose, quiet): + logging.basicConfig( + level=logging.INFO + (quiet - verbose) * 10, + format="[%(levelname)s %(created)f] %(message)s", + ) + ctx.logger = logging.getLogger('autojsdoc') + ctx.visitor = None + ctx.files = [] + ctx.kw = {} + +@autojsdoc.command() +@click.argument('files', type=click.Path(exists=True), nargs=-1) +@click.pass_context +def ast(ctx, files): + """ Prints a structure tree of the provided files + """ + if not files: + print(ctx.get_help()) + visit_files(files, lambda: Printer(level=1), ctx.parent) + +@autojsdoc.command() +@click.option('-m', '--module', multiple=True, help="Only shows dependencies matching any of the patterns") +@click.argument('files', type=click.Path(exists=True), nargs=-1) +@click.pass_context +def dependencies(ctx, module, files): + """ Prints a dot file of all modules to stdout + """ + if not files: + print(ctx.get_help()) + byname = { + mod.name: mod.dependencies + for mod in ABSTRACT_MODULES + } + for modules in visit_files(files, ModuleMatcher, ctx.parent): + for mod in modules: + byname[mod.name] = mod.dependencies + + print('digraph dependencies {') + + todo = set() + # if module filters, roots are only matching modules + if module: + for f in module: + todo.update(fnmatch.filter(byname.keys(), f)) + + for m in todo: + # set a different box for selected roots + print(' "%s" [color=orangered]' % m) + else: + # otherwise check all modules + todo.update(byname) + + done = set() + while todo: + node = todo.pop() + if node in done: + continue + + done.add(node) + deps = byname[node] + todo.update(deps - done) + for dep in deps: + print(' "%s" -> "%s";' % (node, dep)) + print('}') + +try: + autojsdoc.main(prog_name='autojsdoc') +except Exception: + print(cgitb.text(sys.exc_info())) diff --git a/_extensions/autojsdoc/ext/__init__.py b/_extensions/autojsdoc/ext/__init__.py new file mode 100644 index 000000000..e27a9e3d3 --- /dev/null +++ b/_extensions/autojsdoc/ext/__init__.py @@ -0,0 +1,13 @@ +# -*- coding: utf-8 -*- +from .directives import automodule_bound, autodirective_bound +from .extractor import _get_roots + + +def setup(app): + app.add_config_value('js_roots', _get_roots, 'env') + modules = {} + app.add_directive_to_domain('js', 'automodule', automodule_bound(app, modules) + ) + autodirective = autodirective_bound(app, modules) + for n in ['autonamespace', 'automixin', 'autoclass', 'autofunction']: + app.add_directive_to_domain('js', n, autodirective) diff --git a/_extensions/autojsdoc/ext/directives.py b/_extensions/autojsdoc/ext/directives.py new file mode 100644 index 000000000..c666b07a2 --- /dev/null +++ b/_extensions/autojsdoc/ext/directives.py @@ -0,0 +1,719 @@ +# -*- coding: utf-8 -*- +import abc +import collections +import contextlib +import fnmatch +import io +import re + +from docutils import nodes +from docutils.parsers.rst import Directive +from docutils.statemachine import StringList +from sphinx import addnodes +from sphinx.ext.autodoc import members_set_option, bool_option, ALL + +from autojsdoc.ext.extractor import read_js +from ..parser import jsdoc, types + +class DocumenterError(Exception): pass + +@contextlib.contextmanager +def addto(parent, newnode): + assert isinstance(newnode, nodes.Node), \ + "Expected newnode to be a Node, got %s" % (type(newnode)) + yield newnode + parent.append(newnode) + + +def documenter_for(directive, doc): + if isinstance(doc, jsdoc.FunctionDoc): + return FunctionDocumenter(directive, doc) + if isinstance(doc, jsdoc.ClassDoc): + return ClassDocumenter(directive, doc) + if isinstance(doc, jsdoc.MixinDoc): + return MixinDocumenter(directive, doc) + if isinstance(doc, (jsdoc.PropertyDoc, jsdoc.LiteralDoc)): + return PropertyDocumenter(directive, doc) + if isinstance(doc, jsdoc.InstanceDoc): + return InstanceDocumenter(directive, doc) + if isinstance(doc, jsdoc.Unknown): + return UnknownDocumenter(directive, doc) + if isinstance(doc, jsdoc.NSDoc): + return NSDocumenter(directive, doc) + + raise TypeError("No documenter for %s" % type(doc)) + +def to_list(doc, source=None): + return StringList([ + line.rstrip('\n') + for line in io.StringIO(doc) + ], source=source) + +DIRECTIVE_OPTIONS = { + 'members': members_set_option, + 'undoc-members': bool_option, + 'private-members': bool_option, + 'undoc-matches': bool_option, +} +def automodule_bound(app, modules): + class AutoModuleDirective(Directive): + required_arguments = 1 + has_content = True + option_spec = DIRECTIVE_OPTIONS + + # self.state.nested_parse(string, offset, node) => parse context for sub-content (body which can contain RST data) + # => needed for doc (converted?) and for actual directive body + def run(self): + self.env = self.state.document.settings.env + modname = self.arguments[0].strip() + + # TODO: cache/memoize modules & symbols? + if not modules: + read_js(app, modules) + + mods = [ + (name, mod) + for name, mod in modules.items() + if fnmatch.fnmatch(name, modname) + ] + ret = [] + for name, mod in mods: + if mod.is_private: + continue + if name != modname and not (mod.doc or mod.exports): + # this module has no documentation, no exports and was + # not specifically requested through automodule -> skip + # unless requested + if not self.options.get('undoc-matches'): + continue + modsource = mod['sourcefile'] + if modsource: + self.env.note_dependency(modsource) + # not sure what that's used for as normal xrefs are resolved using the id directly + target = nodes.target('', '', ids=['module-' + name], ismod=True) + self.state.document.note_explicit_target(target) + + documenter = ModuleDocumenter(self, mod) + + ret.append(target) + ret.extend(documenter.generate()) + return ret + + return AutoModuleDirective + +def autodirective_bound(app, modules): + documenters = { + 'js:autoclass': ClassDocumenter, + 'js:autonamespace': NSDocumenter, + 'js:autofunction': FunctionDocumenter, + 'js:automixin': MixinDocumenter, + } + class AutoDirective(Directive): + required_arguments = 1 + has_content = True + option_spec = DIRECTIVE_OPTIONS + + def run(self): + self.env = self.state.document.settings.env + + objname = self.arguments[0].strip() + if not modules: + read_js(app, modules) + + # build complete path to object + path = self.env.temp_data.get('autojs:prefix', []) + objname.split('.') + # look for module/object split + for i in range(1, len(path)): + modname, objpath = '.'.join(path[:-i]), path[-i:] + module = modules.get(modname) + if module: + break + else: + raise Exception("Found no valid module in " + '.'.join(path)) + + item = module + # deref' namespaces until we reach the object we're looking for + for k in objpath: + item = item.get_property(k) + + docclass = documenters[self.name] + return docclass(self, item).generate() + + return AutoDirective + +class Documenter(object): + objtype = None + def __init__(self, directive, doc): + self.directive = directive + self.env = directive.env + self.item = doc + + @property + def modname(self): + return self.env.temp_data.get('autojs:module', '') + @property + def classname(self): + return self.env.temp_data.get('autojs:class', '') + def generate(self, all_members=False): + """ + :rtype: List[nodes.Node] + """ + try: + return self._generate(all_members=all_members) + except Exception as e: + raise DocumenterError("Failed to document %s" % self.item) from e + def _generate(self, all_members=False): + objname = self.item.name + prefixed = (self.item['sourcemodule'].name + '.' + objname) if self.item['sourcemodule'] else None + objtype = self.objtype + assert objtype, '%s has no objtype' % type(self) + root = addnodes.desc(domain='js', desctype=objtype, objtype=objtype) + with addto(root, addnodes.desc_signature( + module=self.modname or '', + fullname=objname, + )) as s: + s['class'] = self.classname + + s['ids'] = [] + if objname: + s['ids'].append(objname) + if prefixed: + s['ids'].append(prefixed) + + if objtype: + s += addnodes.desc_annotation( + objtype, objtype, + nodes.Text(' '), + ) + + env = self.env + if objname: + env.domaindata['js']['objects'][objname] = (env.docname, objtype) + if prefixed: + env.domaindata['js']['objects'][prefixed] = (env.docname, objtype) + + # TODO: linkcode_resolve + s += self.make_signature() + with addto(root, addnodes.desc_content()) as c: + # must be here otherwise nested_parse(self.content) will not have + # the prefix set + self.env.temp_data.setdefault('autojs:prefix', []).append(self.item.name) + c += self.make_content(all_members=all_members) + self.env.temp_data['autojs:prefix'].pop() + return [root] + + def make_signature(self): + """ + :rtype: List[nodes.Node] + """ + return [addnodes.desc_name(self.item.name, self.item.name)] + + @abc.abstractmethod + def make_content(self, all_members): + """ + :rtype: List[nodes.Node] + """ + + def document_subtypes(self, subtypes): + docs = [] + with with_mapping_value(self.directive.options, 'undoc-members', True): + for cls in subtypes: + docs += ClassDocumenter(self.directive, cls).generate(all_members=True) + return docs + + +class NSDocumenter(Documenter): + objtype = 'namespace' + def make_content(self, all_members): + doc = self.item + ret = nodes.section() + + if doc.doc: + self.directive.state.nested_parse(to_list(doc.doc), 0, ret) + + self.directive.state.nested_parse(self.directive.content, 0, ret) + + ret += self.document_properties(all_members) + return ret.children + + def should_document(self, member, name, all_members): + """ + :type member: jsdoc.CommentDoc + :type name: str + :type all_members: bool + :rtype: bool + """ + options = self.directive.options + + members = options.get('members') or [] + if not (all_members or members is ALL): + # if a member is requested by name, it's always documented + return name in members + + # ctor params are merged into the class doc + if member.is_constructor: + return False + + # only document "private" members if option is set + if self.is_private(member, name) and not options.get('private-members'): + return False + + # TODO: what if member doesn't have a description but has non-desc tags set? + # TODO: add @public to force documenting symbol? => useful for implicit typedef + return bool(member.doc or options.get('undoc-members')) + + def is_private(self, member, name): + return member.is_private + + def document_properties(self, all_members): + ret = nodes.section() + # TODO: :member-order: [alphabetical | groupwise | bysource] + for (n, p) in self.item.properties: + if not self.should_document(p, n, all_members): + continue + # FIXME: maybe should use property name as name inside? + ret += documenter_for(self.directive, p).generate(all_members=True) + return ret.children + +_NONE = object() +@contextlib.contextmanager +def with_mapping_value(mapping, key, value, restore_to=_NONE): + """ Sets ``key`` to ``value`` for the duration of the context. + + If ``restore_to`` is not provided, restores ``key``'s old value + afterwards, removes it entirely if there was no value for ``key`` in the + mapping. + + .. warning:: for defaultdict & similar mappings, may restore the default + value (depends how the collections' .get behaves) + """ + if restore_to is _NONE: + restore_to = mapping.get(key, _NONE) + mapping[key] = value + try: + yield + finally: + if restore_to is _NONE: + del mapping[key] + else: + mapping[key] = restore_to +class ModuleDocumenter(NSDocumenter): + objtype = 'module' + def document_properties(self, all_members): + with with_mapping_value(self.env.temp_data, 'autojs:module', self.item.name, ''): + return super(ModuleDocumenter, self).document_properties(all_members) + + def make_content(self, all_members): + doc = self.item + content = addnodes.desc_content() + + if doc.exports or doc.dependencies: + with addto(content, nodes.field_list()) as fields: + if doc.exports: + with addto(fields, nodes.field()) as field: + field += nodes.field_name('Exports', 'Exports') + with addto(field, nodes.field_body()) as body: + ref = doc['exports'] # warning: not the same as doc.exports + label = ref or '' + link = addnodes.pending_xref( + ref, nodes.paragraph(ref, label), + refdomain='js', + reftype='any', + reftarget=ref, + ) + link['js:module'] = doc.name + body += link + + if doc.dependencies: + with addto(fields, nodes.field()) as field: + self.make_dependencies(field, doc) + + if doc.doc: + # FIXME: source offset + self.directive.state.nested_parse(to_list(doc.doc, source=doc['sourcefile']), 0, content) + + self.directive.state.nested_parse(self.directive.content, 0, content) + + content += self.document_properties(all_members) + + return content + + def make_dependencies(self, field, doc): + field += nodes.field_name("Depends On", "Depends On") + with addto(field, nodes.field_body()) as body: + with addto(body, nodes.bullet_list()) as deps: + for dep in sorted(doc.dependencies): + ref = addnodes.pending_xref( + dep, nodes.paragraph(dep, dep), + refdomain='js', + reftype='module', + reftarget=dep, + ) + deps += nodes.list_item(dep, ref) + + def should_document(self, member, name, all_members): + # member can be Nothing? + if not member: + return False + modname = getattr(member['sourcemodule'], 'name', None) + doc = self.item + + # always document exported symbol (regardless undoc, private, ...) + # otherwise things become... somewhat odd + if name == doc['exports']: + return True + + # if doc['exports'] the module is exporting a "named" item which + # does not need to be documented twice, if not doc['exports'] it's + # exporting an anonymous item (e.g. object literal) which needs to + # be documented on its own + if name == '' and not doc['exports']: + return True + + # TODO: :imported-members: + # FIXME: *directly* re-exported "foreign symbols"? + return (not modname or modname == doc.name) \ + and super(ModuleDocumenter, self).should_document(member, name, all_members) + + +class ClassDocumenter(NSDocumenter): + objtype = 'class' + + def document_properties(self, all_members): + with with_mapping_value(self.env.temp_data, 'autojs:class', self.item.name, ''): + return super(ClassDocumenter, self).document_properties(all_members) + + def make_signature(self): + sig = super(ClassDocumenter, self).make_signature() + sig.append(self.make_parameters()) + return sig + + def make_parameters(self): + params = addnodes.desc_parameterlist('', '') + ctor = self.item.constructor + if ctor: + params += make_desc_parameters(ctor.params) + + return params + + def make_content(self, all_members): + doc = self.item + ret = nodes.section() + + ctor = self.item.constructor + params = subtypes = [] + if ctor: + check_parameters(self, ctor) + params, subtypes = extract_subtypes(doc.name, ctor) + + fields = nodes.field_list() + fields += self.make_super() + fields += self.make_mixins() + fields += self.make_params(params) + if fields.children: + ret += fields + + if doc.doc: + self.directive.state.nested_parse(to_list(doc.doc), 0, ret) + + self.directive.state.nested_parse(self.directive.content, 0, ret) + + ret += self.document_properties(all_members) + + ret += self.document_subtypes(subtypes) + + return ret.children + + def is_private(self, member, name): + return name.startswith('_') or super(ClassDocumenter, self).is_private(member, name) + + def make_super(self): + doc = self.item + if not doc.superclass: + return [] + + sup_link = addnodes.pending_xref( + doc.superclass.name, nodes.paragraph(doc.superclass.name, doc.superclass.name), + refdomain='js', reftype='class', reftarget=doc.superclass.name, + ) + sup_link['js:module'] = doc.superclass['sourcemodule'].name + return nodes.field( + '', + nodes.field_name("Extends", "Extends"), + nodes.field_body(doc.superclass.name, sup_link), + ) + + def make_mixins(self): + doc = self.item + if not doc.mixins: + return [] + + ret = nodes.field('', nodes.field_name("Mixes", "Mixes")) + with addto(ret, nodes.field_body()) as body: + with addto(body, nodes.bullet_list()) as mixins: + for mixin in sorted(doc.mixins, key=lambda m: m.name): + mixin_link = addnodes.pending_xref( + mixin.name, nodes.paragraph(mixin.name, mixin.name), + refdomain='js', reftype='mixin', reftarget=mixin.name + ) + mixin_link['js:module'] = mixin['sourcemodule'].name + mixins += nodes.list_item('', mixin_link) + return ret + + def make_params(self, params): + if not params: + return [] + + ret = nodes.field('', nodes.field_name('Parameters', 'Parameters')) + with addto(ret, nodes.field_body()) as body,\ + addto(body, nodes.bullet_list()) as holder: + holder += make_parameters(params, mod=self.modname) + return ret + +class InstanceDocumenter(Documenter): + objtype = 'object' + def make_signature(self): + cls = self.item.cls + ret = super(InstanceDocumenter, self).make_signature() + if cls: + super_ref = addnodes.pending_xref( + cls.name, nodes.Text(cls.name, cls.name), + refdomain='js', reftype='class', reftarget=cls.name + ) + super_ref['js:module'] = cls['sourcemodule'].name + ret.append(addnodes.desc_annotation(' instance of ', ' instance of ')) + ret.append(addnodes.desc_type(cls.name, '', super_ref)) + if not ret: + return [addnodes.desc_name('???', '???')] + return ret + + def make_content(self, all_members): + ret = nodes.section() + + if self.item.doc: + self.directive.state.nested_parse(to_list(self.item.doc), 0, ret) + + self.directive.state.nested_parse(self.directive.content, 0, ret) + + return ret.children + +class FunctionDocumenter(Documenter): + @property + def objtype(self): + return 'method' if self.classname else 'function' + def make_signature(self): + ret = super(FunctionDocumenter, self).make_signature() + with addto(ret, addnodes.desc_parameterlist()) as params: + params += make_desc_parameters(self.item.params) + retval = self.item.return_val + if retval.type or retval.doc: + ret.append(addnodes.desc_returns(retval.type or '*', retval.type or '*')) + return ret + + def make_content(self, all_members): + ret = nodes.section() + doc = self.item + + if doc.doc: + self.directive.state.nested_parse(to_list(doc.doc), 0, ret) + + self.directive.state.nested_parse(self.directive.content, 0, ret) + + check_parameters(self, doc) + + params, subtypes = extract_subtypes(self.item.name, self.item) + rdoc = doc.return_val.doc + rtype = doc.return_val.type + if params or rtype or rdoc: + with addto(ret, nodes.field_list()) as fields: + if params: + with addto(fields, nodes.field()) as field: + field += nodes.field_name('Parameters', 'Parameters') + with addto(field, nodes.field_body()) as body,\ + addto(body, nodes.bullet_list()) as holder: + holder.extend(make_parameters(params, mod=doc['sourcemodule'].name)) + if rdoc: + with addto(fields, nodes.field()) as field: + field += nodes.field_name("Returns", "Returns") + with addto(field, nodes.field_body()) as body,\ + addto(body, nodes.paragraph()) as p: + p += nodes.inline(rdoc, rdoc) + if rtype: + with addto(fields, nodes.field()) as field: + field += nodes.field_name("Return Type", "Return Type") + with addto(field, nodes.field_body()) as body, \ + addto(body, nodes.paragraph()) as p: + p += make_types(rtype, mod=doc['sourcemodule'].name) + + ret += self.document_subtypes(subtypes) + + return ret.children + +def pascal_case_ify(name): + """ + Uppercase first letter of ``name``, or any letter following an ``_``. In + the latter case, also strips out the ``_``. + + => key_for becomes KeyFor + => options becomes Options + """ + return re.sub(r'(^|_)\w', lambda m: m.group(0)[-1].upper(), name) +def extract_subtypes(parent_name, doc): + """ Extracts composite parameters (a.b) into sub-types for the parent + parameter, swaps the parent's type from whatever it is to the extracted + one, and returns the extracted type for inclusion into the parent. + + :arg parent_name: name of the containing symbol (function, class), will + be used to compose subtype names + :type parent_name: str + :type doc: FunctionDoc + :rtype: (List[ParamDoc], List[ClassDoc]) + """ + # map of {param_name: [ParamDoc]} (from complete doc) + subparams = collections.defaultdict(list) + for p in map(jsdoc.ParamDoc, doc.get_as_list('param')): + pair = p.name.split('.', 1) + if len(pair) == 2: + k, p.name = pair # remove prefix from param name + subparams[k].append(p) + + # keep original params order as that's the order of formal parameters in + # the function signature + params = collections.OrderedDict((p.name, p) for p in doc.params) + subtypes = [] + # now we can use the subparams map to extract "compound" parameter types + # and swap the new type for the original param's type + for param_name, subs in subparams.items(): + typename = '%s%s' % ( + pascal_case_ify(parent_name), + pascal_case_ify(param_name), + ) + param = params[param_name] + param.type = typename + subtypes.append(jsdoc.ClassDoc({ + 'name': typename, + 'doc': param.doc, + '_members': [ + # TODO: add default value + (sub.name, jsdoc.PropertyDoc(dict(sub.to_dict(), sourcemodule=doc['sourcemodule']))) + for sub in subs + ], + 'sourcemodule': doc['sourcemodule'], + })) + return params.values(), subtypes + +def check_parameters(documenter, doc): + """ + Check that all documented parameters match a formal parameter for the + function. Documented params which don't match the actual function may be + typos. + """ + guessed = set(doc['guessed_params'] or []) + if not guessed: + return + + documented = { + # param name can be of the form [foo.bar.baz=default]\ndescription + jsdoc.ParamDoc(text).name.split('.')[0] + for text in doc.get_as_list('param') + } + odd = documented - guessed + if not odd: + return + + app = documenter.directive.env.app + app.warn("Found documented params %s not in formal parameter list " + "of function %s in module %s (%s)" % ( + ', '.join(odd), + doc.name, + documenter.modname, + doc['sourcemodule']['sourcefile'], + )) + +def make_desc_parameters(params): + for p in params: + if '.' in p.name: + continue + + node = addnodes.desc_parameter(p.name, p.name) + if p.optional: + node = addnodes.desc_optional('', '', node) + yield node + +def make_parameters(params, mod=None): + for param in params: + p = nodes.paragraph('', '', nodes.strong(param.name, param.name)) + if param.default is not None: + p += nodes.Text('=', '=') + p += nodes.emphasis(param.default, param.default) + if param.type: + p += nodes.Text(' (') + p += make_types(param.type, mod=mod) + p += nodes.Text(')') + if param.doc: + p += [ + nodes.Text(' -- '), + nodes.inline(param.doc, param.doc) + ] + yield p + + +def _format_value(v): + if v == '|': + return nodes.emphasis(' or ', ' or ') + if v == ',': + return nodes.Text(', ', ', ') + return nodes.Text(v, v) +def make_types(typespec, mod=None): + # TODO: in closure notation {type=} => optional, do we care? + def format_type(t): + ref = addnodes.pending_xref( + t, addnodes.literal_emphasis(t, t), + refdomain='js', reftype='class', reftarget=t, + ) + if mod: + ref['js:module'] = mod + return ref + + try: + return types.iterate( + types.parse(typespec), + format_type, + _format_value + ) + except ValueError as e: + raise ValueError("%s in '%s'" % (e, typespec)) + + +class MixinDocumenter(NSDocumenter): + objtype = 'mixin' + +class PropertyDocumenter(Documenter): + objtype = 'attribute' + def make_signature(self): + ret = super(PropertyDocumenter, self).make_signature() + proptype = self.item.type + if proptype: + typeref = addnodes.pending_xref( + proptype, nodes.Text(proptype, proptype), + refdomain='js', reftype='class', reftarget=proptype + ) + typeref['js:module'] = self.item['sourcemodule'].name + ret.append(nodes.Text(' ')) + ret.append(typeref) + return ret + + def make_content(self, all_members): + doc = self.item + ret = nodes.section() + + self.directive.state.nested_parse(self.directive.content, 0, ret) + + if doc.doc: + self.directive.state.nested_parse(to_list(doc.doc), 0, ret) + return ret.children + +class UnknownDocumenter(Documenter): + objtype = 'unknown' + def make_content(self, all_members): + return [] diff --git a/_extensions/autojsdoc/ext/extractor.py b/_extensions/autojsdoc/ext/extractor.py new file mode 100644 index 000000000..36211cfd3 --- /dev/null +++ b/_extensions/autojsdoc/ext/extractor.py @@ -0,0 +1,87 @@ +# -*- coding: utf-8 -*- +import io +import os + +import pyjsdoc +import pyjsparser + +try: + from sphinx.util import status_iterator + def it(app): return status_iterator +except ImportError: + # 1.2: Builder.status_iterator + # 1.3: Application.status_iterator (with alias on Builder) + # 1.6: sphinx.util.status_iterator (with *deprecated* aliases on Application and Builder) + # 1.7: removed Application and Builder aliases + # => if no sphinx.util.status_iterator, fallback onto the builder one for + # 1.2 compatibility, remove this entirely if we ever require 1.6+ + status_iterator = None + def it(app): return app.builder.status_iterator + +from ..parser import jsdoc, parser + + +def _get_roots(conf): + env_roots = os.environ.get('AUTOJSDOC_ROOTS_PATH') + if env_roots: + return env_roots.split(':') + return [] + +def read_js(app, modules): + """ + :type app: sphinx.application.Sphinx + :type modules: Dict[str, jsdoc.ModuleDoc] + """ + roots = map(os.path.normpath, app.config.js_roots or [os.path.join(app.confdir, '..')]) + files = [ + os.path.join(r, f) + for root in roots + for r, _, fs in os.walk(root) + if 'static/src/js' in r + for f in fs + if f.endswith('.js') + ] + + modules.update((mod.name, mod) for mod in ABSTRACT_MODULES) + for name in it(app)(files, "Parsing javascript files...", length=len(files)): + with io.open(name) as f: + ast = pyjsparser.parse(f.read()) + modules.update( + (mod.name, mod) + for mod in parser.ModuleMatcher(name).visit(ast) + ) + _resolve_references(modules) + +def _resolve_references(byname): + # must be done in topological order otherwise the dependent can't + # resolve non-trivial references to a dependency properly + for name in pyjsdoc.topological_sort( + *pyjsdoc.build_dependency_graph( + list(byname.keys()), + byname + ) + ): + byname[name].post_process(byname) + +ABSTRACT_MODULES = [ + jsdoc.ModuleDoc({ + 'module': u'web.web_client', + 'dependency': {u'web.AbstractWebClient'}, + 'exports': jsdoc.NSDoc({ + 'name': u'web_client', + 'doc': u'instance of AbstractWebClient', + }), + }), + jsdoc.ModuleDoc({ + 'module': u'web.Tour', + 'dependency': {u'web_tour.TourManager'}, + 'exports': jsdoc.NSDoc({ + 'name': u'Tour', + 'doc': u'maybe tourmanager instance?', + }), + }), + jsdoc.ModuleDoc({ + 'module': u'summernote/summernote', + 'exports': jsdoc.NSDoc({'doc': u"totally real summernote"}), + }) +] diff --git a/_extensions/autojsdoc/parser/__init__.py b/_extensions/autojsdoc/parser/__init__.py new file mode 100644 index 000000000..40a96afc6 --- /dev/null +++ b/_extensions/autojsdoc/parser/__init__.py @@ -0,0 +1 @@ +# -*- coding: utf-8 -*- diff --git a/_extensions/autojsdoc/parser/jsdoc.py b/_extensions/autojsdoc/parser/jsdoc.py new file mode 100644 index 000000000..9e9b0ba44 --- /dev/null +++ b/_extensions/autojsdoc/parser/jsdoc.py @@ -0,0 +1,357 @@ +# -*- coding: utf-8 -*- +import re + +import collections + +import pyjsdoc + +def strip_stars(doc_comment): + """ + Version of jsdoc.strip_stars which always removes 1 space after * if + one is available. + """ + return re.sub('\n\s*?\*[\t ]?', '\n', doc_comment[3:-2]).strip() + +class ParamDoc(pyjsdoc.ParamDoc): + """ + Replace ParamDoc because FunctionDoc doesn't properly handle optional + params or default values (TODO: or compounds) if guessed_params is used + + => augment paramdoc with "required" and "default" items to clean up name + """ + def __init__(self, text): + super(ParamDoc, self).__init__(text) + # param name and doc can be separated by - or :, strip it + self.doc = self.doc.strip().lstrip('-:').lstrip() + self.optional = False + self.default = None + # there may not be a space between the param name and the :, in which + # case the : gets attached to the name, strip *again* + # TODO: formal @param/@property parser to handle this crap properly once and for all + self.name = self.name.strip().rstrip(':') + if self.name.startswith('['): + self.name = self.name.strip('[]') + self.optional = True + if '=' in self.name: + self.name, self.default = self.name.rsplit('=', 1) + def to_dict(self): + d = super(ParamDoc, self).to_dict() + d['optional'] = self.optional + d['default'] = self.default + return d +pyjsdoc.ParamDoc = ParamDoc + +class CommentDoc(pyjsdoc.CommentDoc): + namekey = object() + is_constructor = False + + @property + def name(self): + return self[self.namekey] or self['name'] or self['guessed_name'] + def set_name(self, name): + # not great... + if name != '': + self.parsed['guessed_name'] = name + + @property + def is_private(self): + return 'private' in self.parsed + + def to_dict(self): + d = super(CommentDoc, self).to_dict() + d['name'] = self.name + return d + + # don't resolve already resolved docs (e.g. a literal dict being + # include-ed in two different classes because I don't even care anymore + def become(self, modules): + return self + +class PropertyDoc(CommentDoc): + @classmethod + def from_param(cls, s, sourcemodule=None): + parsed = ParamDoc(s).to_dict() + parsed['sourcemodule'] = sourcemodule + return cls(parsed) + + @property + def type(self): + return self['type'].strip('{}') + + def to_dict(self): + d = super(PropertyDoc, self).to_dict() + d['type'] = self.type + d['is_private'] = self.is_private + return d + +class InstanceDoc(CommentDoc): + @property + def cls(self): + return self['cls'] + + def to_dict(self): + return dict(super(InstanceDoc, self).to_dict(), cls=self.cls) + +class LiteralDoc(CommentDoc): + @property + def type(self): + if self['type']: + return self['type'] + valtype = type(self['value']) + if valtype is bool: + return 'Boolean' + elif valtype is float: + return 'Number' + elif valtype is type(u''): + return 'String' + return '' + + @property + def value(self): + return self['value'] + + def to_dict(self): + d = super(LiteralDoc, self).to_dict() + d['type'] = self.type + d['value'] = self.value + return d + +class FunctionDoc(CommentDoc): + type = 'Function' + namekey = 'function' + + @property + def is_constructor(self): + return self.name == 'init' + + @property + def params(self): + tag_texts = self.get_as_list('param') + # turns out guessed_params is *almost* (?) always set to a list, + # if empty list of guessed params fall back to @params + if not self['guessed_params']: + # only get "primary" params (no "." in name) + return [ + p for p in map(ParamDoc, tag_texts) + if '.' not in p.name + ] + else: + param_dict = {} + for text in tag_texts: + param = ParamDoc(text) + param_dict[param.name] = param + return [param_dict.get(name) or ParamDoc('{} ' + name) + for name in self.get('guessed_params')] + @property + def return_val(self): + ret = self.get('return') or self.get('returns') + type = self.get('type') + if '{' in ret and '}' in ret: + if not '} ' in ret: + # Ensure that name is empty + ret = re.sub(r'\}\s*', '} ', ret) + return ParamDoc(ret) + if ret and type: + return ParamDoc('{%s} %s' % (type, ret)) + return ParamDoc(ret) + + def to_dict(self): + d = super(FunctionDoc, self).to_dict() + d['name'] = self.name + d['params'] = [param.to_dict() for param in self.params] + d['return_val']= self.return_val.to_dict() + return d + +class NSDoc(CommentDoc): + namekey = 'namespace' + def __init__(self, parsed_comment): + super(NSDoc, self).__init__(parsed_comment) + self.members = collections.OrderedDict() + def add_member(self, name, member): + """ + :type name: str + :type member: CommentDoc + """ + member.set_name(name) + self.members[name] = member + + @property + def properties(self): + if self.get('property'): + return [ + (p.name, p) + for p in ( + PropertyDoc.from_param(p, self['sourcemodule']) + for p in self.get_as_list('property') + ) + ] + return list(self.members.items()) or self['_members'] or [] + + def has_property(self, name): + return self.get_property(name) is not None + + def get_property(self, name): + return next((p for n, p in self.properties if n == name), None) + + def to_dict(self): + d = super(NSDoc, self).to_dict() + d['properties'] = [(n, p.to_dict()) for n, p in self.properties] + return d + +class MixinDoc(NSDoc): + namekey = 'mixin' + +class ModuleDoc(NSDoc): + namekey = 'module' + def __init__(self, parsed_comment): + super(ModuleDoc, self).__init__(parsed_comment) + #: callbacks to run with the modules mapping once every module is resolved + self._post_process = [] + + def post_process(self, modules): + for callback in self._post_process: + callback(modules) + + @property + def module(self): + return self # lol + + @property + def dependencies(self): + """ + Returns the immediate dependencies of a module (only those explicitly + declared/used). + """ + return self.get('dependency', None) or set() + + @property + def exports(self): + """ + Returns the actual item exported from the AMD module, can be a + namespace, a class, a function, an instance, ... + """ + return self.get_property('') + + def to_dict(self): + vars = super(ModuleDoc, self).to_dict() + vars['dependencies'] = self.dependencies + vars['exports'] = self.exports + return vars + + def __str__(self): + s = super().__str__() + if self['sourcefile']: + s += " in file " + self['sourcefile'] + return s + +class ClassDoc(NSDoc): + namekey = 'class' + @property + def constructor(self): + return self.get_property('init') + + @property + def superclass(self): + return self['extends'] or self['base'] + + def get_property(self, method_name): + if method_name == 'extend': + return FunctionDoc({ + 'doc': 'Create subclass for %s' % self.name, + 'guessed_function': 'extend', + }) + # FIXME: should ideally be a proxy namespace + if method_name == 'prototype': + return self + return super(ClassDoc, self).get_property(method_name)\ + or (self.superclass and self.superclass.get_property(method_name)) + + @property + def mixins(self): + return self.get_as_list('mixes') + + def to_dict(self): + d = super(ClassDoc, self).to_dict() + d['mixins'] = self.mixins + return d + +DEFAULT = object() +class UnknownNS(NSDoc): + params = () # TODO: log warning when (somehow) trying to access / document an unknown object as ctor? + def get_property(self, name): + return super(UnknownNS, self).get_property(name) or \ + UnknownNS({'name': '{}.{}'.format(self.name, name)}) + + def __getitem__(self, item): + if self._probably_not_property(item): + return super().__getitem__(item) + return self.get_property(item) + + def _probably_not_property(self, item): + return ( + not isinstance(item, str) + or item in (self.namekey, 'name', 'params') + or item.startswith(('_', 'guessed_')) + or item in self.parsed + ) + +class Unknown(CommentDoc): + @classmethod + def from_(cls, source): + def builder(parsed): + inst = cls(parsed) + inst.parsed['source'] = source + return inst + return builder + + @property + def name(self): + return self['name'] + ' ' + self['source'] + + @property + def type(self): + return "Unknown" + + def get_property(self, p): + return Unknown(dict(self.parsed, source=self.name, name=p + '<')) + +def parse_comments(comments, doctype=None): + # find last comment which starts with a * + docstring = next(( + c['value'] + for c in reversed(comments or []) + if c['value'].startswith(u'*') + ), None) or u"" + + # \n prefix necessary otherwise parse_comment fails to take first + # block comment parser strips delimiters, but strip_stars fails without + # them + extract = '\n' + strip_stars('/*' + docstring + '\n*/') + parsed = pyjsdoc.parse_comment(extract, u'') + + if doctype == 'FunctionExpression': + doctype = FunctionDoc + elif doctype == 'ObjectExpression' or doctype is None: + doctype = guess + + if doctype is guess: + return doctype(parsed) + + # in case a specific doctype is given, allow overriding it anyway + return guess(parsed, default=doctype) + +def guess(parsed, default=UnknownNS): + if 'class' in parsed: + return ClassDoc(parsed) + if 'function' in parsed: + return FunctionDoc(parsed) + if 'mixin' in parsed: + return MixinDoc(parsed) + if 'namespace' in parsed: + return NSDoc(parsed) + if 'module' in parsed: + return ModuleDoc(parsed) + if 'type' in parsed: + return PropertyDoc(parsed) + + return default(parsed) diff --git a/_extensions/autojsdoc/parser/parser.py b/_extensions/autojsdoc/parser/parser.py new file mode 100644 index 000000000..626605145 --- /dev/null +++ b/_extensions/autojsdoc/parser/parser.py @@ -0,0 +1,572 @@ +# -*- coding: utf-8 -*- +import collections + +import pyjsdoc + +from . import jsdoc +from . import utils +from .visitor import Visitor, SKIP + +DECLARATOR_INIT_TO_REF = ('Literal', 'Identifier', 'MemberExpression') + + +class ModuleMatcher(Visitor): + """Looks for structures of the form:: + + odoo.define($string, function ($name) { + + These are *Odoo module definitions*, upon encountering one the + matcher: + + * creates a module entry, optionally associated with the module comment + * spawns off a :class:`ModuleBodyMatcher` on the function's body with the + module name and the $name as "require" function + """ + def __init__(self, filename): + super(ModuleMatcher, self).__init__() + self.filename = filename + self.result = [] + def enter_Program(self, node): + pass # allows visiting toplevel + def enter_ExpressionStatement(self, node): + # we're interested in expression statements (toplevel call) + if utils.match(node, {'expression': { + 'callee': { + 'object': {'name': 'odoo'}, + 'property': {'name': 'define'}, + }, + }}): + [module, *_, func] = node['expression']['arguments'] + mod = jsdoc.parse_comments(node.get('comments'), jsdoc.ModuleDoc) + # set module name + mod.set_name(module['value']) + mod.parsed['sourcefile'] = self.filename + self.result.append(mod) + + # get name of require parameter + require = None # a module can have no dependencies + if func['params']: + require = func['params'][0]['name'] + mod.parsed['dependency'], post = ModuleExtractor(mod, require).visit(func['body']) + mod._post_process.extend(post) + # don't recurse since we've fired off a sub-visitor for the + # bits we're interested in + return SKIP + def enter_generic(self, node): + # skip all other toplevel statements + return SKIP + +ref = collections.namedtuple('Ref', 'object property') +def _name(r): + bits = [] + while isinstance(r, ref): + bits.append(str(r.property)) + r = r.object + return '.'.join(reversed(bits)) +def deref(item, prop=None): + assert isinstance(item, ref) + + while isinstance(item, ref): + obj = item.object + if isinstance(obj, ref): + obj = deref(obj) + + if isinstance(obj, (jsdoc.NSDoc, jsdoc.Unknown)): + item = obj.get_property(item.property) + elif isinstance(obj, dict): + item = obj[item.property] + elif isinstance(obj, jsdoc.PropertyDoc): + # f'n dynamic crap + item = jsdoc.Unknown(obj.to_dict()).get_property(item.property) + else: + raise ValueError("%r (%s) should be a dict or namespace" % (obj, type(obj))) + return item + +def m2r(me, scope): + # create a ref to the scope in case it's a hoisted function declaration, + # this may false-positive on other hoisting but w/e + if me['type'] == 'Literal': + return jsdoc.LiteralDoc({'value': me['value']}) + if me['type'] == 'Identifier': + return ref(scope, me['name']) + if me['type'] != 'MemberExpression': + raise ValueError(me) + return ref( + m2r(me['object'], scope), + utils._value(me['property'], strict=True) + ) + +NOTHING = object() +class Declaration(object): + __slots__ = ['id', 'comments'] + def __init__(self, id=None, comments=NOTHING): + self.id = id + self.comments = [] if comments is NOTHING else comments + +class ModuleContent(object): + __slots__ = ['dependencies', 'post'] + def __init__(self, dependencies=NOTHING, post=NOTHING): + self.dependencies = set() if dependencies is NOTHING else dependencies + self.post = [] if post is NOTHING else post + def __iter__(self): + yield self.dependencies + yield self.post + +class Nothing(object): + def __init__(self, name): + self.name = name + def __bool__(self): + return False + __nonzero__ = __bool__ + +class RefProxy(object): + def __init__(self, r): + self._ref = r + def become(self, modules): + s = self + other = deref(self._ref) or Nothing(_name(self._ref)) + # ??? shouldn't all previous refs have been resolved? + if isinstance(other, (RefProxy, ModuleProxy)): + other = other.become(modules) + self.__class__ = other.__class__ + self.__dict__ = other.__dict__ + return self + def set_name(self, name): + pass # ??? +class ModuleProxy(object): + def __init__(self, name): + self._name = name + + # replace the ModuleProxy by the module's exports + def become(self, modules): + s = self + m = modules[self._name].get_property('') or Nothing(self._name) + self.__class__ = m.__class__ + self.__dict__ = m.__dict__ + return self + + def set_name(self, name): + pass # FIXME: ??? + +jq = jsdoc.UnknownNS({ + 'name': u'jQuery', + 'doc': u'', +}) +window = jsdoc.UnknownNS({ + 'doc': '', + 'name': 'window', +}) + +class BaseScope(collections.defaultdict): + """ The base scope assumes anything it's asked for is just an unknown + (global) namespace of some sort. Can hold a bunch of predefined params but + avoids the variables inference system blowing up when new (browser) + globals get used in module bodies. + """ + def __missing__(self, key): + it = jsdoc.UnknownNS({ + 'name': key, + 'doc': u'<%s>' % key, + }) + self[key] = it + return it +BASE_SCOPE = BaseScope(None, { + '_': jsdoc.UnknownNS({'doc': u'', 'name': u'_'}), + '$': jq, 'jQuery': jq, + 'window': window, + 'document': window.get_property('document'), + 'Date': jsdoc.ClassDoc({ + 'name': u'Date', + 'doc': u'', + }), + 'Backbone': jsdoc.UnknownNS({ + '_members': [ + ('Model', jsdoc.ClassDoc({ + 'name': u'Model', + 'doc': u'', + })), + ('Collection', jsdoc.ClassDoc({ + 'name': u'Collection', + 'doc': u'', + })), + ] + }), + 'odoo': jsdoc.UnknownNS({ + 'name': u'odoo', + 'doc': u"Odoo", + '_members': [ + ('name', jsdoc.PropertyDoc({'name': u'csrf_token', 'type': u'{String}'})), + ] + }), + 'undefined': jsdoc.LiteralDoc({'name': u'undefined', 'value': None}), +}) + +class Scope(object): + """ + Add hoc scope versioning/SSA such that rebinding a symbol in a module + scope does not screw everything up e.g. "Foo = Foo.extend({})" should not + have the final Foo extending itself... + """ + def __init__(self, mapping): + self._namemap = self._empty(mapping) + self._targets = [] + for k, v in mapping.items(): + self[k] = v + + @staticmethod + def _empty(mapping): + m = mapping.copy() + m.clear() + return m + + def __setitem__(self, k, v): + self._namemap[k] = len(self._targets) + self._targets.append(v) + + def freeze(self): + d = self._empty(self._namemap) + for k, v in self._namemap.items(): + d[k] = self._targets[v] + return d + +class ModuleExtractor(Visitor): + def __init__(self, module, requirefunc): + super(ModuleExtractor, self).__init__() + self.module = module + self.requirefunc = requirefunc + self.result = ModuleContent() + self.scope = Scope(BASE_SCOPE) + self.declaration = None + + def enter_BlockStatement(self, node): + Hoistifier(self).visit(node) + def exit_BlockStatement(self, node): + for k, v in self.scope._namemap.items(): + if k not in BASE_SCOPE: + self.module.add_member(k, self.scope._targets[v]) + for t in TypedefMatcher(self).visit(node): + self.module.add_member(t.name, t) + + def enter_VariableDeclaration(self, node): + self.declaration = Declaration(comments=node.get('comments')) + def enter_VariableDeclarator(self, node): + # otherwise we've already hoisted the declaration so the variable + # already exist initialised to undefined, and ValueExtractor does not + # handle a None input node so it returns None which makes no sense + if node['init']: + self.declaration.id = node['id']['name'] + self.scope[self.declaration.id] = ValueExtractor( + self, self.declaration + ).visit(node['init'] or []) + self.declaration.id = None + return SKIP + def exit_VariableDeclaration(self, node): + self.declaration = None + + # as the name denotes, AssignmentExpression is an *expression*, which + # means at the module toplevel it is wrapped into an ExpressionStatement + # which is where the comments get attached + def enter_ExpressionStatement(self, node): + self.declaration = Declaration(comments=node.get('comments')) + def exit_ExpressionStatement(self, node): + self.declaration = None + def enter_AssignmentExpression(self, node): + target = node['left'] + if target['type'] == 'Identifier': + self.declaration.id = target['name'] + self.scope[self.declaration.id] = ValueExtractor( + self, self.declaration + ).visit(node['right']) + self.declaration.id = None + return SKIP + + if target['type'] != 'MemberExpression': + raise ValueError("Unhandled assign to %s" % target['type']) + + # only assign to straight a.b.c patterns (OK and trivial literals) + if target['computed'] and target['property']['type'] != 'Literal': + return SKIP + + name = utils._value(target['property'], strict=True) + if isinstance(name, type(u'')) and name.endswith('.extend'): + return SKIP # ignore overwrite of .extend (WTF) + self.declaration.id = name + it = ValueExtractor( + self, self.declaration + ).visit(node['right']) + self.declaration.id = None + assert it, "assigned a non-value from %s to %s" % (node['right'], name) + + @self.result.post.append + def _augment_module(modules): + try: + t = deref(m2r(target['object'], self.scope.freeze())) + except ValueError: + return # f'n extension of global libraries garbage + if not isinstance(t, jsdoc.NSDoc): + # function Foo(){}; Foo.prototype = bar + # fuck that yo + return + # TODO: note which module added this + m = it + if isinstance(m, jsdoc.LiteralDoc): + m = jsdoc.PropertyDoc(m.to_dict()) + t.add_member(name, m) + return SKIP + + def enter_FunctionDeclaration(self, node): + """ Already processed by hoistitifier + """ + return SKIP + + def enter_ReturnStatement(self, node): + self.declaration = Declaration(comments=node.get('comments')) + if node['argument']: + export = ValueExtractor(self, self.declaration).visit(node['argument']) + if isinstance(export, RefProxy): + self.module.parsed['exports'] = _name(export._ref) + self.scope[''] = export + self.declaration = None + return SKIP + + def enter_CallExpression(self, node): + if utils.match(node, { + 'callee': { + 'type': 'MemberExpression', + 'object': lambda n: ( + n['type'] in ('Identifier', 'MemberExpression') + # _.str.include + and not utils._name(n).startswith('_') + ), + 'property': {'name': 'include'}, + } + }): + target = RefProxy(m2r(node['callee']['object'], self.scope.freeze())) + target_name = utils._name(node['callee']['object']) + items = ClassProcessor(self).visit(node['arguments']) + @self.result.post.append + def resolve_extension(modules): + t = target.become(modules) + if not isinstance(t, jsdoc.ClassDoc): + return # FIXME: log warning + # raise ValueError("include() subjects should be classes, %s is %s" % (target_name, type(t))) + # TODO: note which module added these + for it in items: + if isinstance(it, dict): + for n, member in it.items(): + t.add_member(n, member) + else: + t.parsed.setdefault('mixes', []).append(it.become(modules)) + + return SKIP + + def refify(self, node, also=None): + it = m2r(node, self.scope.freeze()) + assert isinstance(it, ref), "Expected ref, got {}".format(it) + px = RefProxy(it) + @self.result.post.append + def resolve(modules): + p = px.become(modules) + if also: also(p) + return px + +class ValueExtractor(Visitor): + def __init__(self, parent, declaration=None): + super(ValueExtractor, self).__init__() + self.parent = parent + self.declaration = declaration or Declaration() + + def enter_generic(self, node): + self.result = jsdoc.parse_comments( + self.declaration.comments, + jsdoc.Unknown.from_(node['type']) + ) + self._update_result_meta() + return SKIP + + def _update_result_meta(self, name=None): + self.result.parsed['sourcemodule'] = self.parent.module + n = name or self.declaration.id + if n: + self.result.set_name(n) + + def enter_Literal(self, node): + self.result = jsdoc.parse_comments( + self.declaration.comments, jsdoc.LiteralDoc) + self._update_result_meta() + self.result.parsed['value'] = node['value'] + return SKIP + def enter_Identifier(self, node): + self.result = self.parent.refify(node) + return SKIP + + def enter_MemberExpression(self, node): + self.result = RefProxy(ref( + ValueExtractor(self.parent).visit(node['object']), + utils._value(node['property'], strict=True) + )) + self.parent.result.post.append(self.result.become) + return SKIP + + def enter_FunctionExpression(self, node): + name, comments = (self.declaration.id, self.declaration.comments) + self.result = jsdoc.parse_comments(comments, jsdoc.FunctionDoc) + self.result.parsed['name'] = node['id'] and node['id']['name'] + self._update_result_meta() + self.result.parsed['guessed_params'] = [p['name'] for p in node['params']] + return SKIP + + def enter_NewExpression(self, node): + comments = self.declaration.comments if self.declaration else node.get('comments') + self.result = ns = jsdoc.parse_comments(comments, jsdoc.InstanceDoc) + self._update_result_meta() + + def _update_contents(cls): + if not isinstance(cls, jsdoc.ClassDoc): + return + ns.parsed['cls'] = cls + self.parent.refify(node['callee'], also=_update_contents) + return SKIP + + def enter_ObjectExpression(self, node): + self.result = obj = jsdoc.parse_comments(self.declaration.comments) + self._update_result_meta() + for n, p in MemberExtractor(parent=self.parent).visit(node['properties']).items(): + obj.add_member(n, p) + return SKIP + + def enter_CallExpression(self, node): + # require(a_module_name) + if utils.match(node, {'callee': {'type': 'Identifier', 'name': self.parent.requirefunc}}): + depname = node['arguments'][0]['value'] + # TODO: clean this up + self.parent.result.dependencies.add(depname) + self.result = ModuleProxy(name=depname) + self.parent.result.post.append(self.result.become) + + # Class.extend(..mixins, {}) + elif utils.match(node, { + 'callee': { + 'type': 'MemberExpression', + 'object': lambda n: ( + n['type'] in ('Identifier', 'MemberExpression') + # $.extend, $.fn.extend, _.extend + and not utils._name(n).startswith(('_', '$')) + ), + 'property': {'name': 'extend'}, + }, + }): # creates a new class, but may not actually return it + obj = node['callee']['object'] + comments = self.declaration.comments + self.result = cls = jsdoc.parse_comments(comments, jsdoc.ClassDoc) + cls.parsed['extends'] = self.parent.refify(obj) + self._update_result_meta() + items = ClassProcessor(self.parent).visit(node['arguments']) + + @self.parent.result.post.append + def add_to_class(modules): + for item in items: + if isinstance(item, dict): + # methods/attributes + for n, method in item.items(): + cls.add_member(n, method) + else: + cls.parsed.setdefault('mixes', []).append(item) + + # other function calls + else: + self.result = jsdoc.parse_comments(self.declaration.comments, jsdoc.guess) + self._update_result_meta() + + return SKIP + +class ClassProcessor(Visitor): + def __init__(self, parent): + super(ClassProcessor, self).__init__() + self.result = [] + self.parent = parent + + def enter_generic(self, node): + self.result.append(self.parent.refify(node)) + return SKIP + def enter_ObjectExpression(self, node): + self.result.append(MemberExtractor(parent=self.parent, for_class=True).visit(node['properties'])) + return SKIP + +String = type(u'') +class MemberExtractor(Visitor): + def __init__(self, parent, for_class=False): + super(MemberExtractor, self).__init__() + self.result = collections.OrderedDict() + self.parent = parent + self.for_class = for_class + + def enter_Property(self, node): + name = utils._value(node['key']) + prop = ValueExtractor( + self.parent, + Declaration(id=name, comments=node.get('comments')) + ).visit(node['value']) + + if isinstance(prop, jsdoc.LiteralDoc): + prop = jsdoc.PropertyDoc(prop.to_dict()) + + # ValueExtractor can return a Ref, maybe this should be sent as part + # of the decl/comments? + if name.startswith('_') and hasattr(prop, 'parsed'): + prop.parsed['private'] = True + self.result[name] = prop + + return SKIP + +class Hoistifier(Visitor): + """ + Processor for variable and function declarations properly hoisting them + to the "top" of a module such that they are available with the relevant + value afterwards. + """ + def __init__(self, parent): + super(Hoistifier, self).__init__() + self.parent = parent + + def enter_generic(self, node): + return SKIP + + # nodes to straight recurse into, others are just skipped + enter_BlockStatement = enter_VariableDeclaration = lambda self, node: None + + def enter_VariableDeclarator(self, node): + self.parent.scope[node['id']['name']] = BASE_SCOPE['undefined'] + + def enter_FunctionDeclaration(self, node): + funcname = node['id']['name'] + self.parent.scope[funcname] = fn = jsdoc.parse_comments( + node.get('comments'), + jsdoc.FunctionDoc, + ) + fn.parsed['sourcemodule'] = self.parent.module + fn.parsed['name'] = funcname + fn.parsed['guessed_params'] = [p['name'] for p in node['params']] + return SKIP + +class TypedefMatcher(Visitor): + def __init__(self, parent): + super(TypedefMatcher, self).__init__() + self.parent = parent + self.result = [] + + enter_BlockStatement = lambda self, node: None + def enter_generic(self, node): + # just traverse all top-level statements, check their comments, and + # bail + for comment in node.get('comments') or []: + if '@typedef' in comment['value']: + extract = '\n' + jsdoc.strip_stars('/*' + comment['value'] + '\n*/') + parsed = pyjsdoc.parse_comment(extract, u'') + p = jsdoc.ParamDoc(parsed['typedef']) + parsed['name'] = p.name + parsed['sourcemodule'] = self.parent.module + # TODO: add p.type as superclass somehow? Builtin types not in scope :( + self.result.append(jsdoc.ClassDoc(parsed)) + + return SKIP diff --git a/_extensions/autojsdoc/parser/tests/README.rst b/_extensions/autojsdoc/parser/tests/README.rst new file mode 100644 index 000000000..dd862beae --- /dev/null +++ b/_extensions/autojsdoc/parser/tests/README.rst @@ -0,0 +1,6 @@ +:orphan: + +These files should be run via pytest_, simply install pytest and from the top +of the Odoo project run ``pytest doc/_extensions``. + +.. _pytest: https://pytest.org/ diff --git a/_extensions/autojsdoc/parser/tests/support.py b/_extensions/autojsdoc/parser/tests/support.py new file mode 100644 index 000000000..f533f98d9 --- /dev/null +++ b/_extensions/autojsdoc/parser/tests/support.py @@ -0,0 +1,73 @@ +# -*- coding: utf-8 -*- +import operator + +import pyjsparser + +from autojsdoc.parser import jsdoc, parser + +params = operator.attrgetter('name', 'type', 'doc') + + +def parse(s, source=None): + tree = pyjsparser.parse(s) + mods = parser.ModuleMatcher(source).visit(tree) + post(mods) + return mods + +def post(mods): + modules = dict(BASE_MODULES) + modules.update((m.name, m) for m in mods) + + for mod in mods: + mod.post_process(modules) + +BASE_MODULES = { + 'other': jsdoc.ModuleDoc({ + 'module': 'other', + '_members': [ + ('', jsdoc.LiteralDoc({'name': 'value', 'value': "ok"})), + ], + }), + 'dep2': jsdoc.ModuleDoc({ + 'module': 'dep2', + '_members': [ + ('', jsdoc.LiteralDoc({'value': 42.})), + ], + }), + 'dep3': jsdoc.ModuleDoc({ + 'module': 'dep3', + '_members': [ + ('', jsdoc.LiteralDoc({'value': 56.})), + ], + }), + 'Class': jsdoc.ModuleDoc({ + 'module': 'Class', + '_members': [ + ('', jsdoc.ClassDoc({ + 'name': 'Class', + 'doc': "Base Class" + })), + ], + }), + 'mixins': jsdoc.ModuleDoc({ + 'module': 'mixins', + '_members': [ + ('', jsdoc.NSDoc({ + 'name': 'mixins', + '_members': [ + ('Bob', jsdoc.ClassDoc({'class': "Bob"})), + ] + })), + ], + }), + 'Mixin': jsdoc.ModuleDoc({ + 'module': 'Mixin', + '_members': [ + ('', jsdoc.MixinDoc({ + '_members': [ + ('a', jsdoc.FunctionDoc({'function': 'a'})), + ] + })), + ], + }) +} diff --git a/_extensions/autojsdoc/parser/tests/test_class.py b/_extensions/autojsdoc/parser/tests/test_class.py new file mode 100644 index 000000000..403ae9b48 --- /dev/null +++ b/_extensions/autojsdoc/parser/tests/test_class.py @@ -0,0 +1,227 @@ +# -*- coding: utf-8 -*- +import pytest + +from autojsdoc.parser import jsdoc +from support import params, parse + +def test_classvar(): + [mod] = parse(""" + odoo.define('a.A', function(require) { + var Class = require('Class'); + /** + * This is my class-kai + */ + var A = Class.extend({}); + return A; + }); + """) + cls = mod.exports + assert type(cls) == jsdoc.ClassDoc + + assert type(cls.superclass) == jsdoc.ClassDoc + assert cls.superclass.name == 'Class' + + assert cls.name == 'A' + assert cls.constructor is None + assert cls.properties == [] + assert cls['doc'] == 'This is my class-kai' + +def test_classret(): + [mod] = parse(""" + odoo.define('a.A', function(require) { + var Class = require('Class'); + /** + * This is my class-kai + */ + return Class.extend({}); + }); + """) + cls = mod.exports + assert type(cls) == jsdoc.ClassDoc + + assert cls.name == '' + assert cls.constructor is None + assert cls.properties == [] + assert cls['doc'] == 'This is my class-kai' + +def test_methods(): + [mod] = parse(""" + odoo.define('a.A', function(require) { + var Class = require('Class'); + return Class.extend({ + /** + * @param {Widget} parent + */ + init: function (parent) {}, + /** + * @returns {Widget} + */ + itself: function () { return this; }, + /** + * @param {MouseEvent} e + */ + _onValidate: function (e) {}, + }); + }); + """) + cls = mod.exports + assert len(cls.properties) == 3 + assert cls.constructor + # assume methods are in source order + [_, init] = cls.properties[0] + assert init == cls.constructor + assert init.name == 'init' + assert not init.is_private + assert init.is_constructor + [param] = init.params + assert params(param) == ('parent', 'Widget', '') + + [_, itself] = cls.properties[1] + assert itself.name == 'itself' + assert not itself.is_private + assert not itself.is_constructor + assert not itself.params + assert params(itself.return_val) == ('', 'Widget', '') + + [_, _on] = cls.properties[2] + assert _on.name == '_onValidate' + assert _on.is_private + assert not _on.is_constructor + [param] = _on.params + assert params(param) == ('e', 'MouseEvent', '') + +def test_mixin_explicit(): + [mod] = parse(""" + odoo.define('a.A', function (require) { + var Class = require('Class'); + var mixins = require('mixins'); + /** + * This is my class-kai + * @mixes mixins.Bob + */ + return Class.extend({}); + }); + """) + cls = mod.exports + # FIXME: ClassDoc may want to m2r(mixin, scope)? + assert cls.mixins == ['mixins.Bob'] + +def test_mixin_implicit(): + [mod] = parse(""" + odoo.define('a.A', function(require) { + var Class = require('Class'); + var Mixin = require('Mixin'); + /** + * This is my class-kai + */ + return Class.extend(Mixin, { foo: function() {} }); + }); + """) + cls = mod.exports + [mixin] = cls.mixins + assert type(mixin) == jsdoc.MixinDoc + assert params(mixin.properties[0][1]) == ('a', 'Function', '') + assert params(mixin.get_property('a')) == ('a', 'Function', '') + + assert params(cls.get_property('foo')) == ('foo', 'Function', '') + +def test_instanciation(): + [A, a] = parse(""" + odoo.define('A', function (r) { + var Class = r('Class'); + /** + * @class A + */ + return Class.extend({ + foo: function () {} + }); + }); + odoo.define('a', function (r) { + var A = r('A'); + var a = new A; + return a; + }); + """) + assert type(a.exports) == jsdoc.InstanceDoc + assert a.exports.cls.name == A.exports.name + +def test_non_function_properties(): + [A] = parse(""" + odoo.define('A', function (r) { + var Class = r('Class'); + return Class.extend({ + template: 'thing', + a_prop: [1, 2, 3], + 'other': {a: 7} + }); + }); + """) + t = A.exports.get_property('template') + assert type(t) == jsdoc.PropertyDoc + assert params(t) == ('template', 'String', '') + assert not t.is_private + +def test_non_extend_classes(): + [mod] = parse(""" + odoo.define('A', function () { + /** + * @class + */ + var Class = function () {} + return Class; + }); + """) + assert type(mod.exports) == jsdoc.ClassDoc + +def test_extend(): + [a, _] = parse(""" + odoo.define('A', function (require) { + var Class = require('Class'); + return Class.extend({}); + }); + odoo.define('B', function (require) { + var A = require('A'); + A.include({ + /** A property */ + a: 3, + /** A method */ + b: function () {} + }); + }); + """) + cls = a.exports + assert type(cls) == jsdoc.ClassDoc + a = cls.get_property('a') + assert type(a) == jsdoc.PropertyDoc + assert params(a) == ('a', 'Number', 'A property') + b = cls.get_property('b') + assert type(b) == jsdoc.FunctionDoc + assert params(b) == ('b', 'Function', 'A method') + +# TODO: also support virtual members? +# TODO: computed properties? +@pytest.mark.skip(reason="Need to implement member/var-parsing?") +def test_members(): + [mod] = parse(""" + odoo.define('A', function (r) { + var Class = r('Class'); + return Class.extend({ + init: function () { + /** + * This is bob + * @var {Foo} + */ + this.foo = 3; + this.bar = 42; + /** + * @member {Baz} + */ + this.baz = null; + } + }); + }); + """) + cls = mod.exports + assert params(cls.members[0]) == ('foo', 'Foo', 'This is bob') + assert params(cls.members[1]) == ('bar', '', '') + assert params(cls.members[2]) == ('baz', 'Baz', '') diff --git a/_extensions/autojsdoc/parser/tests/test_crap.py b/_extensions/autojsdoc/parser/tests/test_crap.py new file mode 100644 index 000000000..70063e6dd --- /dev/null +++ b/_extensions/autojsdoc/parser/tests/test_crap.py @@ -0,0 +1,75 @@ +# -*- coding: utf-8 -*- +""" +Test various crap patterns found in Odoo code to ensure they don't blow up +the parser thingie +""" +from autojsdoc.parser import jsdoc +from support import parse + +def test_export_external(): + [mod] = parse(""" + odoo.define('module', function () { + return $.Deferred().reject(); + }); + """) + assert isinstance(mod.exports, jsdoc.CommentDoc) + assert mod.exports.doc == '' + +def test_extend_jq(): + parse(""" + odoo.define('a', function (r) { + $.extend($.expr[':'], { a: function () {} }); + $.fn.extend({ a: function () {} }); + }); + """) + +def test_extend_dynamic(): + parse(""" + odoo.define('a', function () { + foo.bar.baz[qux + '_external'] = function () {}; + }); + """) + +def test_extend_deep(): + parse(""" + odoo.define('a', function () { + var eventHandler = $.summernote.eventHandler; + var dom = $.summernote.core.dom; + dom.thing = function () {}; + + var fn_editor_currentstyle = eventHandler.modules.editor.currentStyle; + eventHandler.modules.editor.currentStyle = function () {} + }); + """) + +def test_arbitrary(): + parse(""" + odoo.define('bob', function () { + var page = window.location.href.replace(/^.*\/\/[^\/]+/, ''); + var mailWidgets = ['mail_followers', 'mail_thread', 'mail_activity', 'kanban_activity']; + var bob; + var fldj = foo.getTemplate().baz; + }); + """) + +def test_prototype(): + [A, B] = parse(""" + odoo.define('mod1', function () { + var exports = {}; + exports.Foo = Backbone.Model.extend({}); + exports.Bar = Backbone.Model.extend({}); + var BarCollection = Backbone.Collection.extend({ + model: exports.Bar, + }); + exports.Baz = Backbone.Model.extend({}); + return exports; + }); + odoo.define('mod2', function (require) { + var models = require('mod1'); + var _super_orderline = models.Bar.prototype; + models.Foo = models.Bar.extend({}); + var _super_order = models.Baz.prototype; + models.Bar = models.Baz.extend({}); + }); + """) + diff --git a/_extensions/autojsdoc/parser/tests/test_module.py b/_extensions/autojsdoc/parser/tests/test_module.py new file mode 100644 index 000000000..2a63cc175 --- /dev/null +++ b/_extensions/autojsdoc/parser/tests/test_module.py @@ -0,0 +1,173 @@ +# -*- coding: utf-8 -*- + +from autojsdoc.parser import jsdoc +from support import params, parse, BASE_MODULES + + +def test_single(): + [mod] = parse(""" + /** + * This is a super module! + */ + odoo.define('supermodule', function (req) { + var other = req('other'); + }); + """) + assert mod.name == 'supermodule' + assert mod.dependencies == {'other'} + assert mod.exports is None + assert mod.doc == "This is a super module!" + m = mod.get_property('other') + assert m.name == 'value', "property is exported value not imported module" + +def test_multiple(): + [mod1, mod2, mod3] = parse(""" + odoo.define('module1', function (req) { + return 1; + }); + odoo.define('module2', function (req) { + return req('dep2'); + }); + odoo.define('module3', function (req) { + var r = req('dep3'); + return r; + }); + """) + assert mod1.name == 'module1' + assert mod1.dependencies == set() + assert isinstance(mod1.exports, jsdoc.LiteralDoc) + assert mod1.exports.value == 1.0 + assert mod1.exports['sourcemodule'] is mod1 + assert mod1.doc == "" + + assert mod2.name == 'module2' + assert mod2.dependencies == {'dep2'} + assert isinstance(mod2.exports, jsdoc.LiteralDoc) + assert mod2.exports.value == 42.0 + assert mod2.doc == "" + + assert mod3.name == 'module3' + assert mod3.dependencies == {'dep3'} + assert isinstance(mod3.exports, jsdoc.LiteralDoc) + assert mod3.exports.value == 56.0 + assert mod3.doc == '' + +def test_func(): + [mod] = parse(""" + odoo.define('module', function (d) { + /** + * @param {Foo} bar this is a bar + * @param {Baz} qux this is a qux + */ + return function (bar, qux) { + return 42; + } + }); + """) + exports = mod.exports + assert type(exports) == jsdoc.FunctionDoc + assert exports['sourcemodule'] is mod + + assert exports.name == '' + assert exports.is_constructor == False + assert exports.is_private == False + + assert params(exports.params[0]) == ('bar', 'Foo', 'this is a bar') + assert params(exports.params[1]) == ('qux', 'Baz', 'this is a qux') + assert params(exports.return_val) == ('', '', '') + +def test_hoist(): + [mod] = parse(""" + odoo.define('module', function() { + return foo; + /** + * @param a_thing + */ + function foo(a_thing) { + return 42; + } + }); + """) + actual = mod.exports + assert type(actual) == jsdoc.FunctionDoc + [param] = actual.params + assert params(param) == ('a_thing', '', '') + +def test_export_instance(): + [mod] = parse(""" + odoo.define('module', function (require) { + var Class = require('Class'); + /** + * Provides an instance of Class + */ + return new Class(); + }); + """) + assert type(mod.exports) == jsdoc.InstanceDoc + assert mod.exports.doc == 'Provides an instance of Class' + assert mod.exports['sourcemodule'] is mod + +def test_bounce(): + [m2, m1] = parse(""" + odoo.define('m2', function (require) { + var Item = require('m1'); + return { + Item: Item + }; + }); + odoo.define('m1', function (require) { + var Class = require('Class'); + var Item = Class.extend({}); + return Item; + }); + """) + assert type(m2.exports) == jsdoc.NSDoc + it = m2.exports.get_property('Item') + assert type(it) == jsdoc.ClassDoc + assert it['sourcemodule'] is m1 + assert sorted([n for n, _ in m1.properties]) == ['', 'Class', 'Item'] + +def test_reassign(): + [m] = parse(""" + odoo.define('m', function (require) { + var Class = require('Class'); + /** local class */ + var Class = Class.extend({}); + return Class + }); + """) + assert m.exports.doc == 'local class' + # can't use equality or identity so use class comment... + assert m.exports.superclass.doc == 'Base Class' + +def test_attr(): + [m1, m2] = parse(""" + odoo.define('m1', function (require) { + var Class = require('Class'); + var Item = Class.extend({}); + return {Item: Item}; + }); + odoo.define('m2', function (require) { + var Item = require('m1').Item; + Item.include({}); + return Item.extend({}); + }); + """) + assert type(m2.exports) == jsdoc.ClassDoc + # that these two are resolved separately may be an issue at one point (?) + assert m2.exports.superclass.to_dict() == m1.exports.get_property('Item').to_dict() + +def test_nothing_implicit(): + [m] = parse(""" + odoo.define('m', function () { + }); + """) + assert m.exports is None + +def test_nothing_explicit(): + [m] = parse(""" + odoo.define('m', function () { + return; + }); + """) + assert m.exports is None diff --git a/_extensions/autojsdoc/parser/tests/test_namespace.py b/_extensions/autojsdoc/parser/tests/test_namespace.py new file mode 100644 index 000000000..54cbcf8cb --- /dev/null +++ b/_extensions/autojsdoc/parser/tests/test_namespace.py @@ -0,0 +1,177 @@ +# -*- coding: utf-8 -*- +from autojsdoc.parser import jsdoc +from support import parse, params + + +def test_empty(): + [mod] = parse(""" + odoo.define('a.ns', function (r) { + return {}; + }); + """) + assert type(mod.exports) == jsdoc.NSDoc + assert mod.exports.properties == [] + +def test_inline(): + [mod] = parse(""" + odoo.define('a.ns', function (r) { + return { + /** + * a thing + * @type {Boolean} + */ + a: true + } + }); + """) + assert isinstance(mod.exports, jsdoc.NSDoc) + [(n, p)] = mod.exports.properties + assert n == 'a' + assert params(p) == ('a', 'Boolean', 'a thing') + +def test_header(): + [mod] = parse(""" + odoo.define('a.ns', function (r) { + /** + * @property {Boolean} a a thing + */ + return { a: true } + }); + """) + assert isinstance(mod.exports, jsdoc.NSDoc) + [(_, p)] = mod.exports.properties + assert params(p) == ('a', 'Boolean', 'a thing') + +def test_header_conflict(): + """ should the header or the inline comment take precedence? """ + [mod] = parse(""" + odoo.define('a.ns', function (r) { + /** + * @property {Boolean} a a thing + */ + return { + /** @type {String} */ + a: true + } + }); + """) + assert isinstance(mod.exports, jsdoc.NSDoc) + [(_, p)] = mod.exports.properties + assert params(p) == ('a', 'Boolean', 'a thing') + +def test_mixin(): + [mod] = parse(""" + odoo.define('a.mixin', function (r) { + /** + * @mixin + */ + return { + /** + * @returns {Number} a number + */ + do_thing: function other() { return 42; } + } + }); + """) + assert isinstance(mod.exports, jsdoc.MixinDoc) + [(n, p)] = mod.exports.properties + assert n == 'do_thing' + assert params(p) == ('other', 'Function', '') + assert params(p.return_val) == ('', 'Number', 'a number') + +def test_literal(): + [mod] = parse(""" + odoo.define('a.ns', function (r) { + /** whop whop */ + return { + 'a': 1, + /** wheee */ + 'b': 2, + }; + }); + """) + assert mod.exports.doc == 'whop whop' + [(_1, a), (_2, b)] = mod.exports.properties + assert params(a) == ('a', 'Number', '') + assert params(b) == ('b', 'Number', 'wheee') + +def test_fill_ns(): + [mod] = parse(""" + odoo.define('a.ns', function (r) { + var Class = r('Class'); + var ns = {}; + /** ok */ + ns.a = 1; + /** @type {String} */ + ns['b'] = 2; + /** Ike */ + ns.c = Class.extend({}); + ns.d = function () {} + return ns; + }); + """) + ns = mod.exports + assert type(ns) == jsdoc.NSDoc + [(_a, a), (_b, b), (_c, c), (_d, d)] = ns.properties + assert params(a) == ('a', 'Number', 'ok') + assert params(b) == ('b', 'String', '') + assert type(c) == jsdoc.ClassDoc + assert type(d) == jsdoc.FunctionDoc + +def test_extend_other(): + [o, b] = parse(""" + odoo.define('a.ns', function () { + /** @name outer */ + return { + /** @name inner */ + a: {} + }; + }); + odoo.define('b', function (r) { + var o = r('a.ns'); + var Class = r('Class'); + /** Class 1 */ + o.a.b = Class.extend({m_b: 1}); + /** Class 2 */ + o.a['c'] = Class.extend({m_c: 1}); + }); + """) + [(_, m)] = o.exports.properties + assert type(m) == jsdoc.NSDoc + + b = m.get_property('b') + assert type(b) == jsdoc.ClassDoc + assert b.get_property('m_b') + assert b.doc == 'Class 1' + + c = m.get_property('c') + assert type(c) == jsdoc.ClassDoc + assert c.get_property('m_c') + assert c.doc == 'Class 2' + +def test_ns_variables(): + [mod] = parse(""" + odoo.define('A', function (r) { + var Class = r('Class'); + var Thing = Class.extend({}); + return { + Thing: Thing + }; + }); + """) + p = mod.exports.get_property('Thing') + assert type(p) == jsdoc.ClassDoc + +def test_diff(): + """ Have the NS key and the underlying object differ + """ + [mod] = parse(""" + odoo.define('mod', function (r) { + var Class = r('Class'); + var Foo = Class.extend({}); + return { Class: Foo }; + }); + """) + c = mod.exports.get_property('Class') + assert type(c) == jsdoc.ClassDoc + assert c.name == 'Foo' diff --git a/_extensions/autojsdoc/parser/tests/test_params.py b/_extensions/autojsdoc/parser/tests/test_params.py new file mode 100644 index 000000000..c51139b8c --- /dev/null +++ b/_extensions/autojsdoc/parser/tests/test_params.py @@ -0,0 +1,116 @@ +# -*- coding: utf-8 -*- +from autojsdoc.parser.jsdoc import ParamDoc + +"Lorem ipsum dolor sit amet, consectetur adipiscing elit." + +def test_basic(): + d = ParamDoc("Lorem ipsum dolor sit amet, consectetur adipiscing elit.").to_dict() + assert d == { + 'name': 'Lorem', + 'type': '', + 'optional': False, + 'default': None, + 'doc': 'ipsum dolor sit amet, consectetur adipiscing elit.', + } + + d = ParamDoc("{Lorem} ipsum dolor sit amet, consectetur adipiscing elit.").to_dict() + assert d == { + 'name': 'ipsum', + 'type': 'Lorem', + 'optional': False, + 'default': None, + 'doc': 'dolor sit amet, consectetur adipiscing elit.', + } + +def test_optional(): + d = ParamDoc("[Lorem] ipsum dolor sit amet, consectetur adipiscing elit.").to_dict() + assert d == { + 'name': 'Lorem', + 'type': '', + 'optional': True, + 'default': None, + 'doc': 'ipsum dolor sit amet, consectetur adipiscing elit.', + } + + d = ParamDoc("[Lorem=42] ipsum dolor sit amet, consectetur adipiscing elit.").to_dict() + assert d == { + 'name': 'Lorem', + 'type': '', + 'optional': True, + 'default': "42", + 'doc': 'ipsum dolor sit amet, consectetur adipiscing elit.', + } + + d = ParamDoc("{Lorem} [ipsum] dolor sit amet, consectetur adipiscing elit.").to_dict() + assert d == { + 'name': 'ipsum', + 'type': 'Lorem', + 'optional': True, + 'default': None, + 'doc': 'dolor sit amet, consectetur adipiscing elit.', + } + + d = ParamDoc("{Lorem} [ipsum=42] dolor sit amet, consectetur adipiscing elit.").to_dict() + assert d == { + 'name': 'ipsum', + 'type': 'Lorem', + 'optional': True, + 'default': '42', + 'doc': 'dolor sit amet, consectetur adipiscing elit.', + } + +def test_returns(): + d = ParamDoc("{} ipsum dolor sit amet, consectetur adipiscing elit.").to_dict() + assert d == { + 'name': '', + 'type': '', + 'optional': False, + 'default': None, + 'doc': 'ipsum dolor sit amet, consectetur adipiscing elit.', + } + d = ParamDoc("{Lorem} ipsum dolor sit amet, consectetur adipiscing elit.").to_dict() + assert d == { + 'name': '', + 'type': 'Lorem', + 'optional': False, + 'default': None, + 'doc': 'ipsum dolor sit amet, consectetur adipiscing elit.', + } + +def test_odd(): + d = ParamDoc("{jQuery} [$target] the node where content will be prepended").to_dict() + assert d == { + 'name': '$target', + 'type': 'jQuery', + 'optional': True, + 'default': None, + 'doc': 'the node where content will be prepended', + } + + d = ParamDoc("""{htmlString} [content] DOM element, + array of elements, HTML string or jQuery object to prepend to $target""").to_dict() + assert d == { + 'name': 'content', + 'type': 'htmlString', + 'optional': True, + 'default': None, + 'doc': "DOM element,\n array of elements, HTML string or jQuery object to prepend to $target", + } + + d = ParamDoc("{Boolean} [options.in_DOM] true if $target is in the DOM").to_dict() + assert d == { + 'name': 'options.in_DOM', + 'type': 'Boolean', + 'optional': True, + 'default': None, + 'doc': 'true if $target is in the DOM', + } + + d = ParamDoc("Lorem\n ipsum dolor sit amet, consectetur adipiscing elit.").to_dict() + assert d['doc'] == 'ipsum dolor sit amet, consectetur adipiscing elit.' + + d = ParamDoc("Lorem - ipsum dolor sit amet, consectetur adipiscing elit.").to_dict() + assert d['doc'] == 'ipsum dolor sit amet, consectetur adipiscing elit.' + + d = ParamDoc("Lorem : ipsum dolor sit amet, consectetur adipiscing elit.").to_dict() + assert d['doc'] == 'ipsum dolor sit amet, consectetur adipiscing elit.' diff --git a/_extensions/autojsdoc/parser/tests/test_typespec.py b/_extensions/autojsdoc/parser/tests/test_typespec.py new file mode 100644 index 000000000..b2a5a5258 --- /dev/null +++ b/_extensions/autojsdoc/parser/tests/test_typespec.py @@ -0,0 +1,104 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +import pytest + +from autojsdoc.parser import types + +def test_parser(): + assert types.parse("MouseEvent|TouchEvent") == types.Alt([ + types.Type('MouseEvent', []), + types.Type('TouchEvent', []), + ]) + + assert types.parse('Deferred') == types.Alt([ + types.Type('Deferred', [ + types.Alt([types.Type('Object', [])]) + ]) + ]) + + assert types.parse('Object|undefined') == types.Alt([ + types.Type('Object', []), + types.Literal('undefined'), + ]) + assert types.parse('any[]') == types.Alt([ + types.Type('Array', [types.Type('any', [])]) + ]) + assert types.parse("'xml' | 'less'") == types.Alt([ + types.Literal('xml'), + types.Literal('less'), + ]) + assert types.parse('Foo') == types.Alt([ + types.Type('Foo', [types.Alt([ + types.Type('Bar', []), + types.Type('Array', [types.Type('Bar', [])]), + types.Literal('null'), + ])]) + ]) + assert types.parse('Function>') == types.Alt([ + types.Type('Function', [types.Alt([ + types.Type('Array', [types.Alt([ + types.Type('Array', [ + types.Type('Object', []) + ]) + ])]) + ])]) + ]) + + +def test_tokens(): + toks = list(types.tokenize('A')) + assert toks == [(types.NAME, 'A')] + + toks = list(types.tokenize('"foo" | "bar"')) + assert toks == [(types.LITERAL, 'foo'), (types.OP, '|'), (types.LITERAL, 'bar')] + + toks = list(types.tokenize('1 or 2')) + assert toks == [(types.LITERAL, 1), (types.OP, '|'), (types.LITERAL, 2)] + + toks = list(types.tokenize('a.b.c.d')) + assert toks == [ + (types.NAME, 'a'), (types.OP, '.'), + (types.NAME, 'b'), (types.OP, '.'), + (types.NAME, 'c'), (types.OP, '.'), + (types.NAME, 'd'), + ] + + toks = list(types.tokenize('Function')) + assert toks == [ + (types.NAME, 'Function'), + (types.OP, '<'), + (types.NAME, 'String'), + (types.OP, ','), + (types.NAME, 'Object'), + (types.OP, '>') + ] + + toks = list(types.tokenize('Function>')) + assert toks == [ + (types.NAME, 'Function'), + (types.OP, '<'), + (types.NAME, 'Array'), + (types.OP, '<'), + (types.NAME, 'Object'), + (types.OP, '['), + (types.OP, ']'), + (types.OP, '>'), + (types.OP, '>') + ] + +def test_peekable(): + p = types.Peekable(range(5)) + + assert p.peek() == 0 + assert next(p) == 0, "consuming should yield previously peeked value" + next(p) + next(p) + assert next(p) == 3 + assert p.peek() == 4 + next(p) + assert p.peek(None) is None + with pytest.raises(StopIteration): + p.peek() + with pytest.raises(StopIteration): + next(p) diff --git a/_extensions/autojsdoc/parser/types.py b/_extensions/autojsdoc/parser/types.py new file mode 100644 index 000000000..69836f46a --- /dev/null +++ b/_extensions/autojsdoc/parser/types.py @@ -0,0 +1,196 @@ +# -*- coding: utf-8 -*- +import ast +import collections +import io +import token +from tokenize import generate_tokens + +LITERAL_NAME = ('null', 'undefined', 'true', 'false') + +__all__ = ['tokenize', 'parse', 'iterate', 'Alt', 'Type', 'Literal'] + +class _Marker(object): + __slots__ = ['name'] + def __init__(self, name): self.name = name + def __repr__(self): return '<%s>' % self.name + +OP = _Marker('OP') +NAME = _Marker('NAME') +LITERAL = _Marker('LITERAL') +def tokenize(typespec): + typespec = typespec.replace('$', 'jQuery') + for toktype, string, _, _, _ in generate_tokens(io.StringIO(typespec).readline): + if toktype == token.NAME: + if string == 'or': # special case "A or B" + # TODO: deprecation warning + yield (OP, '|') + else: + yield (NAME, string) + elif toktype == token.OP: + if string in '|<>[].,': + yield (OP, string) + elif string == '>>': + yield (OP, '>') + yield (OP, '>') + elif string == '*': # maybe? + yield (NAME, 'any') + elif string in '()': + # TODO: deprecation warning + # seems useless unless we add support for "..." + continue + else: + raise ValueError("Unknown typespec operator %r" % string) + elif toktype in (token.STRING, token.NUMBER): # enum-ish + yield (LITERAL, ast.literal_eval(string)) + elif toktype == token.ENDMARKER: + return + elif toktype == token.NEWLINE: + pass + else: + raise ValueError("Unknown typespec token %s" % token.tok_name[toktype]) + +Alt = collections.namedtuple('Alt', ['params']) +Literal = collections.namedtuple('Literal', ['value']) +Type = collections.namedtuple('Type', ['name', 'params']) + +def iterate(t, fortype, forliteral): + if isinstance(t, Alt): + ps = Peekable(t.params) + for param in ps: + for it in iterate(param, fortype, forliteral): + yield it + if ps.peek(None): + yield forliteral('|') + elif isinstance(t, Type): + yield fortype(t.name) + if not t.params: + return + yield forliteral('<') + ps = Peekable(t.params) + for param in ps: + for it in iterate(param, fortype, forliteral): + yield it + if ps.peek(None): + yield forliteral(',') + yield forliteral('>') + elif isinstance(t, Literal): + yield forliteral(t.value) + else: + raise ValueError('Unknown item %s' % t) + +def parse(typespec): + tokens = Peekable(tokenize(typespec)) + return parse_alt(tokens) + +# alt = param +# | param '|' alt +def parse_alt(tokens): + alt = Alt([]) + while True: + alt.params.append(parse_param(tokens)) + if not next_is(tokens, (OP, '|')): + break + return alt + +# param = 'null' | 'undefined' | 'true' | 'false' +# | number +# | string +# | typename '[]' +# X '[' alt ']' +# | typename '<' params '>' +# params = alt | alt ',' params +def parse_param(tokens): + t, v = tokens.peek() + if t == LITERAL or (t == NAME and v in LITERAL_NAME): + next(tokens) + return Literal(str(v)) + + # # [typespec] # should this be Array or an n-uple allowing multiple items? + # if tok == (OP, '['): + # next(tokens) + # t = Type('Array', [parse_alt(tokens)]) + # rest = next(tokens) + # expect(rest, (OP, ']')) + # return t + + t = Type(parse_typename(tokens), []) + # type[] + peek = tokens.peek(None) + if peek == (OP, '['): + next(tokens) # skip peeked + expect(next(tokens), (OP, ']')) + return Type('Array', [t]) + # type + if peek == (OP, '<'): + next(tokens) # skip peeked + while True: + t.params.append(parse_alt(tokens)) + n = next(tokens) + if n == (OP, ','): + continue + if n == (OP, '>'): + break + raise ValueError("Expected OP ',' or OP ',', got %s '%s'" % n) + return t + +# typename = name | name '.' typename +def parse_typename(tokens): + typename = [] + while True: + (t, n) = next(tokens) + if t != NAME: + raise ValueError("Expected a name token, got %s '%s'" % (t, n)) + if n in LITERAL_NAME: + raise ValueError("Expected a type name, got literal %s" % n) + typename.append(n) + if not next_is(tokens, (OP, '.')): + break + return '.'.join(typename) + +def next_is(tokens, expected): + """ + Consumes the next token if it's `expected` otherwise does not touch the + tokens. + + Returns whether it consumed a token + """ + if tokens.peek(None) == expected: + next(tokens) + return True + return False + +def expect(actual, expected): + """ Raises ValueError if `actual` and `expected` are different + + :type actual: (object, str) + :type expected: (object, str) + """ + if actual != expected: + raise ValueError("Expected %s '%s', got %s '%s'" % (expected + actual)) + + +NONE = object() +class Peekable(object): + __slots__ = ['_head', '_it'] + def __init__(self, iterable): + self._head = NONE + self._it = iter(iterable) + + def __iter__(self): + return self + def __next__(self): + if self._head is not NONE: + r, self._head = self._head, NONE + return r + return next(self._it) + next = __next__ + + def peek(self, default=NONE): + if self._head is NONE: + try: + self._head = next(self._it) + except StopIteration: + if default is not NONE: + return default + raise + return self._head diff --git a/_extensions/autojsdoc/parser/utils.py b/_extensions/autojsdoc/parser/utils.py new file mode 100644 index 000000000..213327f6c --- /dev/null +++ b/_extensions/autojsdoc/parser/utils.py @@ -0,0 +1,295 @@ +# -*- coding: utf-8 -*- +def _name(node): + if node['type'] == 'Identifier': + return node['name'] + if node['type'] == 'MemberExpression': + return "%s.%s" % (_name(node['object']), _name(node['property'])) + raise ValueError("Unnamable node type %s" % node['type']) + +def _value(node, strict=False): + t = node['type'] + if t == 'Identifier': + return node['name'] + elif t == 'Literal': + return node['value'] + msg = '<%s has no value>' % _identify(node, {}) + if strict: + raise ValueError(msg) + return msg + +def _identify(valnode, ns): + if valnode is None: + return "None" + # already identified and re-set in the ns? + if isinstance(valnode, Printable): + return valnode + + # check other non-empty returns + t = valnode['type'] + if t == "Literal": + if valnode.get('regex'): + return valnode['regex']['pattern'] + return valnode['value'] + elif t == "Identifier": + n = valnode['name'] + return ns.get(n, Global(n)) + elif t == "NewExpression": + return Instance(_identify(valnode['callee'], ns)) + elif t == "ObjectExpression": + return Namespace({ + _value(prop['key']): _identify(prop['value'], ns) + for prop in valnode['properties'] + }) + elif t == "MemberExpression": + return Deref( + _identify(valnode['object'], ns), + _identify(valnode['property'], ns) if valnode['computed'] else valnode['property']['name'], + ) + elif t == "CallExpression": + return Call( + _identify(valnode['callee'], ns), + [_identify(arg, ns) for arg in valnode['arguments']], + ) + elif t == 'BinaryExpression': + return "%s %s %s" % ( + _identify(valnode['left'], ns), + valnode['operator'], + _identify(valnode['right'], ns), + ) + else: + return t + +class Counter(object): + __slots__ = ['_number'] + def __init__(self): + self._number = 0 + + def __bool__(self): + return bool(self._number) + __nonzero__ = __bool__ + def __int__(self): + return self._number + + def get(self): + return self._number + def increment(self): + self._number += 1 + return self._number + def decrement(self): + self._number -= 1 + return self._number + + def __enter__(self): + self._number += 1 + return self._number + def __exit__(self, *args): + self._number -= 1 + +class Printable(object): + __slots__ = [] + depth = Counter() + def __repr__(self): + with self.depth as i: + if i > 2: + return "%s(...)" % type(self).__name__ + return "%s(%s)" % (type(self).__name__, ', '.join( + "%s=%r" % (k, getattr(self, k)) + for k in self.__slots__ + if not k.startswith('_') + if getattr(self, k) + )) + + +def resolve(obj, store): + if obj is None: + return None + + if isinstance(obj, (type(u''), bool, int, float)): + return obj + + if isinstance(obj, ModuleProxy): + return resolve(obj.get(), store) + + try: + if getattr(obj, 'resolved', False): + return obj + return obj.resolve(store) + except AttributeError: + raise TypeError("Unresolvable {!r}".format(obj)) + +class Resolvable(Printable): + """ + For types resolved in place + """ + __slots__ = ['resolved'] + + def __init__(self): + super(Resolvable, self).__init__() + self.resolved = False + def resolve(self, store): + self.resolved = True + return self + +class Namespace(Resolvable): + __slots__ = ['attrs'] + def __init__(self, attrs): + super(Namespace, self).__init__() + self.attrs = attrs + def resolve(self, store): + r = super(Namespace, self).resolve(store) + self.attrs = { + k: resolve(v, store) + for k, v in self.attrs.items() + } + return r + def __getitem__(self, key): + assert isinstance(key, type(u'')), "%r is not a namespace key" % key + try: + return self.attrs[key] + except KeyError: + return Global(self)[key] + +class Module(Resolvable): + __slots__ = ('name', 'comments', 'exports', 'augments', 'dependencies', '_store') + def __init__(self, name, store, comments=()): + super(Module, self).__init__() + self.name = name + self._store = store + self.comments = tuple(comments) + self.exports = None + self.augments = [] + self.dependencies = set() + store[name] = self + def add_dependency(self, depname): + dep = ModuleProxy(depname, self._store) + self.dependencies.add(dep) + return dep + def get(self): + return self + def resolve(self, store=None): + r = super(Module, self).resolve(store) + self.exports = resolve(self.exports, self._store) + self.augments = [resolve(a, self._store) for a in self.augments] + self.dependencies = { + resolve(d, self._store) + for d in self.dependencies + } + return r + def __getitem__(self, k): + try: + return self.exports[k] + except KeyError: + return Global(self)[k] + +class ModuleProxy(object): + def __init__(self, name, store): + self.name = name + self.store = store + def get(self): + # web.web_client is not an actual module + return self.store.get(self.name, '' % self.name) + def __repr__(self): + return repr(self.get()) + def __hash__(self): + return hash(self.name) + def __eq__(self, other): + return self.name == other.name + +class Class(Resolvable): + __slots__ = ['name', 'members', 'extends', 'mixins', 'comments'] + def __init__(self, name, extends): + super(Class, self).__init__() + self.name = name + self.extends = extends + self.members = {} + self.mixins = [] + self.comments = [] + + def resolve(self, store): + r = super(Class, self).resolve(store) + self.extends = resolve(self.extends, store) + self.mixins = [resolve(m, store) for m in self.mixins] + return r + +class Instance(Resolvable): + __slots__ = ['type'] + def __init__(self, type): + super(Instance, self).__init__() + self.type = type + + def resolve(self, store): + r = super(Instance, self).resolve(store) + self.type = resolve(self.type, store) + if isinstance(self.type, Module): + self.type = self.type.exports + return r + + def __getitem__(self, key): + return Global(self)['key'] + +class Function(Resolvable): + __slots__ = ['comments'] + def __init__(self, comments): + super(Function, self).__init__() + self.comments = comments + +class Deref(Printable): + __slots__ = ['object', 'property'] + def __init__(self, object, property): + self.object = object + self.property = property + def resolve(self, store): + return resolve(self.object, store)[self.property] + +class Call(Resolvable): + __slots__ = ['callable'] + def __init__(self, callable, arguments): + super(Call, self).__init__() + self.callable = callable + def resolve(self, store): + r = super(Call, self).resolve(store) + self.callable = resolve(self.callable, store) + return r + def __getitem__(self, item): + return Global(self)[item] + +def get(node, it, *path): + if not path: + return node[it] + return get(node[it], *path) + + +def match(node, pattern): + """Checks that ``pattern`` is a subset of ``node``. + + If a sub-pattern is a callable it will be called with the current + sub-node and its result is the match's. Other sub-patterns are + checked by equality against the correspoding sub-node. + + Can be used to quickly check the descendants of a node of suitable + type. + + TODO: support checking list content? Would be convenient to + constraint e.g. CallExpression based on their parameters + + """ + if node is None and pattern is not None: + return + if callable(pattern): + return pattern(node) + if isinstance(node, list): + return all(len(node) > i and match(node[i], v) for i, v in pattern.items()) + if isinstance(pattern, dict): + return all(match(node.get(k), v) for k, v in pattern.items()) + return node == pattern + + +class Global(object): + def __init__(self, name): + self.name = name + def __repr__(self): + return '' % self.name + def __getitem__(self, name): + return Global('%s.%s' % (self.name, name)) + def resolve(self, store): + return self diff --git a/_extensions/autojsdoc/parser/visitor.py b/_extensions/autojsdoc/parser/visitor.py new file mode 100644 index 000000000..e5529d511 --- /dev/null +++ b/_extensions/autojsdoc/parser/visitor.py @@ -0,0 +1,126 @@ +from pyjsparser.pyjsparserdata import Syntax + +_binary = lambda n: [n['left'], n['right']] +_function = lambda n: n['params'] + n['defaults'] + [n['body']] +# yields children in visitation order +_children = { + Syntax.ArrayExpression: lambda n: n['elements'], + Syntax.ArrayPattern: lambda n: n['elements'], + Syntax.ArrowFunctionExpression: _function, + Syntax.AssignmentExpression: _binary, + Syntax.AssignmentPattern: _binary, + Syntax.BinaryExpression: _binary, + Syntax.BlockStatement: lambda n: n['body'], + Syntax.BreakStatement: lambda n: [], + Syntax.CallExpression: lambda n: [n['callee']] + n['arguments'], + Syntax.CatchClause: lambda n: [n['param'], n['body']], + Syntax.ClassBody: lambda n: [n['body']], + Syntax.ClassDeclaration: lambda n: [n['superClass'], n['body']], + Syntax.ClassExpression: lambda n: [n['superClass'], n['body']], + Syntax.ConditionalExpression: lambda n: [n['test'], n['consequent'], n['alternate']], + Syntax.ContinueStatement: lambda n: [], + Syntax.DebuggerStatement: lambda n: [], + Syntax.DoWhileStatement: lambda n: [n['body'], n['test']], + Syntax.EmptyStatement: lambda n: [], + Syntax.ExportAllDeclaration: lambda n: [n['source']], + Syntax.ExportDefaultDeclaration: lambda n: [n['declaration']], + Syntax.ExportNamedDeclaration: lambda n: ([n['declaration']] if n['declaration'] else n['specifiers']) + [n['source']], + Syntax.ExportSpecifier: lambda n: [n['local'], n['exported']], + Syntax.ExpressionStatement: lambda n: [n['expression']], + Syntax.ForStatement: lambda n: [n['init'], n['test'], n['update'], n['body']], + Syntax.ForInStatement: lambda n: [n['left'], n['right'], n['body']], + Syntax.FunctionDeclaration: _function, + Syntax.FunctionExpression: _function, + Syntax.Identifier: lambda n: [], + Syntax.IfStatement: lambda n: [n['test'], n['consequent'], n['alternate']], + Syntax.ImportDeclaration: lambda n: n['specifiers'] + [n['source']], + Syntax.ImportDefaultSpecifier: lambda n: [n['local']], + Syntax.ImportNamespaceSpecifier: lambda n: [n['local']], + Syntax.ImportSpecifier: lambda n: [n['local'], n['imported']], + Syntax.LabeledStatement: lambda n: [n['body']], + Syntax.Literal: lambda n: [], + Syntax.LogicalExpression: _binary, + Syntax.MemberExpression: lambda n: [n['object'], n['property']], + #Syntax.MethodDefinition: lambda n: [], + Syntax.NewExpression: lambda n: [n['callee']] + n['arguments'], + Syntax.ObjectExpression: lambda n: n['properties'], + Syntax.ObjectPattern: lambda n: n['properties'], + Syntax.Program: lambda n: n['body'], + Syntax.Property: lambda n: [n['key'], n['value']], + Syntax.RestElement: lambda n: [n['argument']], + Syntax.ReturnStatement: lambda n: [n['argument']], + Syntax.SequenceExpression: lambda n: n['expressions'], + Syntax.SpreadElement: lambda n: [n['argument']], + Syntax.Super: lambda n: [], + Syntax.SwitchCase: lambda n: [n['test'], n['consequent']], + Syntax.SwitchStatement: lambda n: [n['discriminant']] + n['cases'], + Syntax.TaggedTemplateExpression: lambda n: [n['tag'], n['quasi']], + Syntax.TemplateElement: lambda n: [], + Syntax.TemplateLiteral: lambda n: n['quasis'] + n['expressions'], + Syntax.ThisExpression: lambda n: [], + Syntax.ThrowStatement: lambda n: [n['argument']], + Syntax.TryStatement: lambda n: [n['block'], n['handler'], n['finalizer']], + Syntax.UnaryExpression: lambda n: [n['argument']], + Syntax.UpdateExpression: lambda n: [n['argument']], + Syntax.VariableDeclaration: lambda n: n['declarations'], + Syntax.VariableDeclarator: lambda n: [n['id'], n['init']], + Syntax.WhileStatement: lambda n: [n['test'], n['body']], + Syntax.WithStatement: lambda n: [n['object'], n['body']], +} + +SKIP = object() +class Visitor(object): + """ + Generic visitor for the pyjsparser AST. + + Visitation is driven by the ``visit`` method, which iterates the tree in + depth-first pre-order. + + For each node, calls ``enter_$NODETYPE``, visits the children then calls + ``exit_$NODETYPE``. If the enter or exit methods are not present on the + visitor, falls back on ``enter_generic`` and ``exit_generic``. + + Any ``enter_`` method can return ``SKIP`` to suppress both the traversal + of the subtree *and* the call to the corresponding ``exit_`` method + (whether generic or specific). + + For convenience, ``visit`` will return whatever is set as the visitor's + ``result`` attribute, ``None`` by default. + + ``visit`` can be given multiple root nodes, and it can be called multiple + times. The ``result`` attribute is cleared at each call but not between + two roots of the same ``visit`` call. + """ + def __init__(self): + super(Visitor, self).__init__() + self.result = None + + def enter_generic(self, node): pass + def exit_generic(self, node): pass + + def visit(self, nodes): + if isinstance(nodes, dict): + nodes = [nodes] + # if multiple nodes are passed in, we need to reverse the order in + # order to traverse front-to-back rather than the other way around + nodes = list(reversed(nodes)) + + while nodes: + node = nodes.pop() + # should probably filter None descendants in _children... + if node is None: + continue + node_type = node['type'] + if node_type == '_exit': + node = node['node'] + getattr(self, 'exit_' + node['type'], self.exit_generic)(node) + continue + + if getattr(self, 'enter_' + node_type, self.enter_generic)(node) is SKIP: + continue + + nodes.append({'type': '_exit', 'node': node}) + nodes.extend(reversed(_children[node_type](node))) + + return self.result +