Jedi 0.11 parser + test fixes (#777)

* Basic tokenizer

* Fixed property names

* Tests, round I

* Tests, round II

* tokenizer test

* Remove temorary change

* Fix merge issue

* Merge conflict

* Merge conflict

* Completion test

* Fix last line

* Fix javascript math

* Make test await for results

* Add license headers

* Rename definitions to types

* License headers

* Fix typo in completion details (typo)

* Fix hover test

* Russian translations

* Update to better translation

* Fix typo

*  #70 How to get all parameter info when filling in a function param list

* Fix #70 How to get all parameter info when filling in a function param list

* Clean up

* Clean imports

* CR feedback

* Trim whitespace for test stability

* More tests

* Better handle no-parameters documentation

* Better handle ellipsis and Python3

* #385 Auto-Indentation doesn't work after comment

* #141 Auto indentation broken when return keyword involved

* Undo changes

* #627 Docstrings for builtin methods are not parsed correctly

* reStructuredText converter

* Fix: period is not an operator

* Minor fixes

* Restructure

* Tests

* Tests

* Code heuristics

* Baselines

* HTML handling

* Lists

* State machine

* Baselines

* Squash

* no message

* Whitespace difference

* Update Jedi to 0.11.1

* Enable Travis

* Test fixes

* Undo change

* Jedi 0.11 with parser

* Undo changes

* Undo changes

* Test fixes

* More tests

* Tests

Author: Mikhail Arkhipov

.vscode/settings.json

@@ -18,6 +18,5 @@
     "python.linting.enabled": false,
     "python.unitTest.promptToConfigure": false,
     "python.workspaceSymbols.enabled": false,
-    "python.formatting.provider": "none",
-    "files.insertFinalNewline": true
-}
+    "python.formatting.provider": "none"
+}

pythonFiles/completion.py

@@ -15,7 +15,7 @@ def __init__(self, new_stdout=None):
 
     def __enter__(self):
         sys.stdout.flush()
-        oldstdout_fno = self.oldstdout_fno = os.dup(sys.stdout.fileno())
+        self.oldstdout_fno = os.dup(sys.stdout.fileno())
         os.dup2(self._new_stdout.fileno(), 1)
 
     def __exit__(self, exc_type, exc_value, traceback):
@@ -47,7 +47,6 @@ def __init__(self):
             self.drive_mount = ''
 
     def _get_definition_type(self, definition):
-        is_built_in = definition.in_builtin_module
         # if definition.type not in ['import', 'keyword'] and is_built_in():
         #    return 'builtin'
         try:
@@ -89,7 +88,7 @@ def _generate_signature(self, completion):
             return ''
         return '%s(%s)' % (
             completion.name,
-            ', '.join(p.description for p in completion.params if p))
+            ', '.join(p.description[6:] for p in completion.params if p))
 
     def _get_call_signatures(self, script):
         """Extract call signatures from jedi.api.Script object in failsafe way.
@@ -108,18 +107,28 @@ def _get_call_signatures(self, script):
             for pos, param in enumerate(signature.params):
                 if not param.name:
                     continue
+
+                name = self._get_param_name(param)
                 if param.name == 'self' and pos == 0:
                     continue
-                try:
-                    name, value = param.description.split('=')
-                except ValueError:
-                    name = param.description
-                    value = None
                 if name.startswith('*'):
                     continue
+
+                value = self._get_param_value(param)
                 _signatures.append((signature, name, value))
         return _signatures
 
+    def _get_param_name(self, p):
+        if(p.name.startswith('param ')):
+            return p.name[6:] # drop leading 'param '
+        return p.name
+
+    def _get_param_value(self, p):
+        pair = p.description.split('=')
+        if(len(pair) > 1):
+            return pair[1]
+        return None
+
     def _get_call_signatures_with_args(self, script):
         """Extract call signatures from jedi.api.Script object in failsafe way.
 
@@ -150,16 +159,12 @@ def _get_call_signatures_with_args(self, script):
             for pos, param in enumerate(signature.params):
                 if not param.name:
                     continue
+
+                name = self._get_param_name(param)
                 if param.name == 'self' and pos == 0:
                     continue
-                try:
-                    name, value = param.description.split('=')
-                except ValueError:
-                    name = param.description
-                    value = None
-                # if name.startswith('*'):
-                #    continue
-                #_signatures.append((signature, name, value))
+
+                value = self._get_param_value(param)
                 paramDocstring = ''
                 try:
                     paramDocstring = param.docstring()
@@ -251,8 +256,7 @@ def _serialize_methods(self, script, identifier=None, prefix=''):
         for completion in completions:
             params = []
             if hasattr(completion, 'params'):
-                params = [p.description for p in completion.params
-                          if ARGUMENT_RE.match(p.description)]
+                params = [p.description for p in completion.params if p]
             if completion.parent().type == 'class':
               _methods.append({
                 'parent': completion.parent().name,
@@ -288,50 +292,8 @@ def _top_definition(self, definition):
                 return d
         return definition
 
-    def _extract_range_jedi_0_9_0(self, definition):
-        from jedi import common
-        from jedi.parser.utils import load_parser
-        # get the scope range
-        try:
-            if definition.type in ['class', 'function'] and hasattr(definition, '_definition'):
-                scope = definition._definition
-                start_line = scope.start_pos[0] - 1
-                start_column = scope.start_pos[1]
-                end_line = scope.end_pos[0] - 1
-                end_column = scope.end_pos[1]
-                # get the lines
-                path = definition._definition.get_parent_until().path
-                parser = load_parser(path)
-                lines = common.splitlines(parser.source)
-                lines[end_line] = lines[end_line][:end_column]
-                # trim the lines
-                lines = lines[start_line:end_line + 1]
-                lines = '\n'.join(lines).rstrip().split('\n')
-                end_line = start_line + len(lines) - 1
-                end_column = len(lines[-1]) - 1
-            else:
-                symbol = definition._name
-                start_line = symbol.start_pos[0] - 1
-                start_column = symbol.start_pos[1]
-                end_line = symbol.end_pos[0] - 1
-                end_column =  symbol.end_pos[1]
-            return {
-                'start_line': start_line,
-                'start_column': start_column,
-                'end_line': end_line,
-                'end_column': end_column
-            }
-        except Exception as e:
-            return {
-                'start_line': definition.line - 1,
-                'start_column': definition.column, 
-                'end_line': definition.line - 1,
-                'end_column': definition.column
-            }
-
-    def _extract_range_jedi_0_10_1(self, definition):
-        from jedi import common
-        from jedi.parser.python import parse
+    def _extract_range_jedi_0_11_1(self, definition):
+        from parso.utils import split_lines
         # get the scope range
         try:
             if definition.type in ['class', 'function']:
@@ -341,7 +303,7 @@ def _extract_range_jedi_0_10_1(self, definition):
                 start_column = scope.start_pos[1]
                 # get the lines
                 code = scope.get_code(include_prefix=False)
-                lines = common.splitlines(code)
+                lines = split_lines(code)
                 # trim the lines
                 lines = '\n'.join(lines).rstrip().split('\n')
                 end_line = start_line + len(lines) - 1
@@ -380,10 +342,7 @@ def _extract_range(self, definition):
         last character of actual code. That's why we extract the lines that
         make up our scope and trim the trailing whitespace.
         """
-        if jedi.__version__ in ('0.9.0', '0.10.0'):
-            return self._extract_range_jedi_0_9_0(definition)
-        else:
-            return self._extract_range_jedi_0_10_1(definition)
+        return self._extract_range_jedi_0_11_1(definition)
 
     def _get_definitionsx(self, definitions, identifier=None, ignoreNoModulePath=False):
         """Serialize response to be read from VSCode.
@@ -680,22 +639,17 @@ def watch(self):
 if __name__ == '__main__':
     cachePrefix = 'v'
     modulesToLoad = ''
-    if len(sys.argv) > 0 and sys.argv[1] == 'preview':
-        jediPath = os.path.join(os.path.dirname(__file__), 'preview')
-        jediPreview = True
-        if len(sys.argv) > 2:
-            modulesToLoad = sys.argv[2]
-    elif len(sys.argv) > 0 and sys.argv[1] == 'custom':
+    if len(sys.argv) > 2 and sys.argv[1] == 'custom':
         jediPath = sys.argv[2]
         jediPreview = True
         cachePrefix = 'custom_v'
         if len(sys.argv) > 3:
             modulesToLoad = sys.argv[3]
     else:
-        #std
-        jediPath = os.path.join(os.path.dirname(__file__), 'release')
-        if len(sys.argv) > 2:
-            modulesToLoad = sys.argv[2]
+        #release
+        jediPath = os.path.dirname(__file__)
+        if len(sys.argv) > 1:
+            modulesToLoad = sys.argv[1]
 
     sys.path.insert(0, jediPath)
     import jedi

pythonFiles/release/jedi/__init__.py renamed to pythonFiles/jedi/__init__.py

@@ -0,0 +1,58 @@
+r"""
+Parso is a Python parser that supports error recovery and round-trip parsing
+for different Python versions (in multiple Python versions). Parso is also able
+to list multiple syntax errors in your python file.
+
+Parso has been battle-tested by jedi_. It was pulled out of jedi to be useful
+for other projects as well.
+
+Parso consists of a small API to parse Python and analyse the syntax tree.
+
+.. _jedi: https://github.com/davidhalter/jedi
+
+A simple example:
+
+>>> import parso
+>>> module = parso.parse('hello + 1', version="3.6")
+>>> expr = module.children[0]
+>>> expr
+PythonNode(arith_expr, [<Name: hello@1,0>, <Operator: +>, <Number: 1>])
+>>> print(expr.get_code())
+hello + 1
+>>> name = expr.children[0]
+>>> name
+<Name: hello@1,0>
+>>> name.end_pos
+(1, 5)
+>>> expr.end_pos
+(1, 9)
+
+To list multiple issues:
+
+>>> grammar = parso.load_grammar()
+>>> module = grammar.parse('foo +\nbar\ncontinue')
+>>> error1, error2 = grammar.iter_errors(module)
+>>> error1.message
+'SyntaxError: invalid syntax'
+>>> error2.message
+"SyntaxError: 'continue' not properly in loop"
+"""
+
+from parso.parser import ParserSyntaxError
+from parso.grammar import Grammar, load_grammar
+from parso.utils import split_lines, python_bytes_to_unicode
+
+
+__version__ = '0.1.1'
+
+
+def parse(code=None, **kwargs):
+    """
+    A utility function to avoid loading grammars.
+    Params are documented in :py:meth:`parso.Grammar.parse`.
+
+    :param str version: The version used by :py:func:`parso.load_grammar`.
+    """
+    version = kwargs.pop('version', None)
+    grammar = load_grammar(version=version)
+    return grammar.parse(code, **kwargs)

pythonFiles/preview/jedi/__main__.py renamed to pythonFiles/jedi/__main__.py

@@ -0,0 +1,103 @@
+"""
+To ensure compatibility from Python ``2.6`` - ``3.3``, a module has been
+created. Clearly there is huge need to use conforming syntax.
+"""
+import sys
+import platform
+
+# Cannot use sys.version.major and minor names, because in Python 2.6 it's not
+# a namedtuple.
+py_version = int(str(sys.version_info[0]) + str(sys.version_info[1]))
+
+# unicode function
+try:
+    unicode = unicode
+except NameError:
+    unicode = str
+
+is_pypy = platform.python_implementation() == 'PyPy'
+
+
+def use_metaclass(meta, *bases):
+    """ Create a class with a metaclass. """
+    if not bases:
+        bases = (object,)
+    return meta("HackClass", bases, {})
+
+
+try:
+    encoding = sys.stdout.encoding
+    if encoding is None:
+        encoding = 'utf-8'
+except AttributeError:
+    encoding = 'ascii'
+
+
+def u(string):
+    """Cast to unicode DAMMIT!
+    Written because Python2 repr always implicitly casts to a string, so we
+    have to cast back to a unicode (and we now that we always deal with valid
+    unicode, because we check that in the beginning).
+    """
+    if py_version >= 30:
+        return str(string)
+
+    if not isinstance(string, unicode):
+        return unicode(str(string), 'UTF-8')
+    return string
+
+
+try:
+    FileNotFoundError = FileNotFoundError
+except NameError:
+    FileNotFoundError = IOError
+
+
+def utf8_repr(func):
+    """
+    ``__repr__`` methods in Python 2 don't allow unicode objects to be
+    returned. Therefore cast them to utf-8 bytes in this decorator.
+    """
+    def wrapper(self):
+        result = func(self)
+        if isinstance(result, unicode):
+            return result.encode('utf-8')
+        else:
+            return result
+
+    if py_version >= 30:
+        return func
+    else:
+        return wrapper
+
+
+try:
+    from functools import total_ordering
+except ImportError:
+    # Python 2.6
+    def total_ordering(cls):
+        """Class decorator that fills in missing ordering methods"""
+        convert = {
+            '__lt__': [('__gt__', lambda self, other: not (self < other or self == other)),
+                       ('__le__', lambda self, other: self < other or self == other),
+                       ('__ge__', lambda self, other: not self < other)],
+            '__le__': [('__ge__', lambda self, other: not self <= other or self == other),
+                       ('__lt__', lambda self, other: self <= other and not self == other),
+                       ('__gt__', lambda self, other: not self <= other)],
+            '__gt__': [('__lt__', lambda self, other: not (self > other or self == other)),
+                       ('__ge__', lambda self, other: self > other or self == other),
+                       ('__le__', lambda self, other: not self > other)],
+            '__ge__': [('__le__', lambda self, other: (not self >= other) or self == other),
+                       ('__gt__', lambda self, other: self >= other and not self == other),
+                       ('__lt__', lambda self, other: not self >= other)]
+        }
+        roots = set(dir(cls)) & set(convert)
+        if not roots:
+            raise ValueError('must define at least one ordering operation: < > <= >=')
+        root = max(roots)       # prefer __lt__ to __le__ to __gt__ to __ge__
+        for opname, opfunc in convert[root]:
+            if opname not in roots:
+                opfunc.__name__ = opname
+                opfunc.__doc__ = getattr(int, opname).__doc__
+                setattr(cls, opname, opfunc)
+        return cls