diff --git a/.vscode/settings.json b/.vscode/settings.json index 17a82485a5ad..be66f967c5c4 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -18,6 +18,5 @@ "python.linting.enabled": false, "python.unitTest.promptToConfigure": false, "python.workspaceSymbols.enabled": false, - "python.formatting.provider": "none", - "files.insertFinalNewline": true -} + "python.formatting.provider": "none" +} \ No newline at end of file diff --git a/pythonFiles/completion.py b/pythonFiles/completion.py index 4a0f915fd145..e530be32b367 100644 --- a/pythonFiles/completion.py +++ b/pythonFiles/completion.py @@ -15,7 +15,7 @@ def __init__(self, new_stdout=None): def __enter__(self): sys.stdout.flush() - oldstdout_fno = self.oldstdout_fno = os.dup(sys.stdout.fileno()) + self.oldstdout_fno = os.dup(sys.stdout.fileno()) os.dup2(self._new_stdout.fileno(), 1) def __exit__(self, exc_type, exc_value, traceback): @@ -47,7 +47,6 @@ def __init__(self): self.drive_mount = '' def _get_definition_type(self, definition): - is_built_in = definition.in_builtin_module # if definition.type not in ['import', 'keyword'] and is_built_in(): # return 'builtin' try: @@ -89,7 +88,7 @@ def _generate_signature(self, completion): return '' return '%s(%s)' % ( completion.name, - ', '.join(p.description for p in completion.params if p)) + ', '.join(p.description[6:] for p in completion.params if p)) def _get_call_signatures(self, script): """Extract call signatures from jedi.api.Script object in failsafe way. @@ -108,18 +107,28 @@ def _get_call_signatures(self, script): for pos, param in enumerate(signature.params): if not param.name: continue + + name = self._get_param_name(param) if param.name == 'self' and pos == 0: continue - try: - name, value = param.description.split('=') - except ValueError: - name = param.description - value = None if name.startswith('*'): continue + + value = self._get_param_value(param) _signatures.append((signature, name, value)) return _signatures + def _get_param_name(self, p): + if(p.name.startswith('param ')): + return p.name[6:] # drop leading 'param ' + return p.name + + def _get_param_value(self, p): + pair = p.description.split('=') + if(len(pair) > 1): + return pair[1] + return None + def _get_call_signatures_with_args(self, script): """Extract call signatures from jedi.api.Script object in failsafe way. @@ -150,16 +159,12 @@ def _get_call_signatures_with_args(self, script): for pos, param in enumerate(signature.params): if not param.name: continue + + name = self._get_param_name(param) if param.name == 'self' and pos == 0: continue - try: - name, value = param.description.split('=') - except ValueError: - name = param.description - value = None - # if name.startswith('*'): - # continue - #_signatures.append((signature, name, value)) + + value = self._get_param_value(param) paramDocstring = '' try: paramDocstring = param.docstring() @@ -251,8 +256,7 @@ def _serialize_methods(self, script, identifier=None, prefix=''): for completion in completions: params = [] if hasattr(completion, 'params'): - params = [p.description for p in completion.params - if ARGUMENT_RE.match(p.description)] + params = [p.description for p in completion.params if p] if completion.parent().type == 'class': _methods.append({ 'parent': completion.parent().name, @@ -288,50 +292,8 @@ def _top_definition(self, definition): return d return definition - def _extract_range_jedi_0_9_0(self, definition): - from jedi import common - from jedi.parser.utils import load_parser - # get the scope range - try: - if definition.type in ['class', 'function'] and hasattr(definition, '_definition'): - scope = definition._definition - start_line = scope.start_pos[0] - 1 - start_column = scope.start_pos[1] - end_line = scope.end_pos[0] - 1 - end_column = scope.end_pos[1] - # get the lines - path = definition._definition.get_parent_until().path - parser = load_parser(path) - lines = common.splitlines(parser.source) - lines[end_line] = lines[end_line][:end_column] - # trim the lines - lines = lines[start_line:end_line + 1] - lines = '\n'.join(lines).rstrip().split('\n') - end_line = start_line + len(lines) - 1 - end_column = len(lines[-1]) - 1 - else: - symbol = definition._name - start_line = symbol.start_pos[0] - 1 - start_column = symbol.start_pos[1] - end_line = symbol.end_pos[0] - 1 - end_column = symbol.end_pos[1] - return { - 'start_line': start_line, - 'start_column': start_column, - 'end_line': end_line, - 'end_column': end_column - } - except Exception as e: - return { - 'start_line': definition.line - 1, - 'start_column': definition.column, - 'end_line': definition.line - 1, - 'end_column': definition.column - } - - def _extract_range_jedi_0_10_1(self, definition): - from jedi import common - from jedi.parser.python import parse + def _extract_range_jedi_0_11_1(self, definition): + from parso.utils import split_lines # get the scope range try: if definition.type in ['class', 'function']: @@ -341,7 +303,7 @@ def _extract_range_jedi_0_10_1(self, definition): start_column = scope.start_pos[1] # get the lines code = scope.get_code(include_prefix=False) - lines = common.splitlines(code) + lines = split_lines(code) # trim the lines lines = '\n'.join(lines).rstrip().split('\n') end_line = start_line + len(lines) - 1 @@ -380,10 +342,7 @@ def _extract_range(self, definition): last character of actual code. That's why we extract the lines that make up our scope and trim the trailing whitespace. """ - if jedi.__version__ in ('0.9.0', '0.10.0'): - return self._extract_range_jedi_0_9_0(definition) - else: - return self._extract_range_jedi_0_10_1(definition) + return self._extract_range_jedi_0_11_1(definition) def _get_definitionsx(self, definitions, identifier=None, ignoreNoModulePath=False): """Serialize response to be read from VSCode. @@ -680,22 +639,17 @@ def watch(self): if __name__ == '__main__': cachePrefix = 'v' modulesToLoad = '' - if len(sys.argv) > 0 and sys.argv[1] == 'preview': - jediPath = os.path.join(os.path.dirname(__file__), 'preview') - jediPreview = True - if len(sys.argv) > 2: - modulesToLoad = sys.argv[2] - elif len(sys.argv) > 0 and sys.argv[1] == 'custom': + if len(sys.argv) > 2 and sys.argv[1] == 'custom': jediPath = sys.argv[2] jediPreview = True cachePrefix = 'custom_v' if len(sys.argv) > 3: modulesToLoad = sys.argv[3] else: - #std - jediPath = os.path.join(os.path.dirname(__file__), 'release') - if len(sys.argv) > 2: - modulesToLoad = sys.argv[2] + #release + jediPath = os.path.dirname(__file__) + if len(sys.argv) > 1: + modulesToLoad = sys.argv[1] sys.path.insert(0, jediPath) import jedi diff --git a/pythonFiles/release/jedi/__init__.py b/pythonFiles/jedi/__init__.py old mode 100755 new mode 100644 similarity index 100% rename from pythonFiles/release/jedi/__init__.py rename to pythonFiles/jedi/__init__.py diff --git a/pythonFiles/preview/jedi/__main__.py b/pythonFiles/jedi/__main__.py similarity index 100% rename from pythonFiles/preview/jedi/__main__.py rename to pythonFiles/jedi/__main__.py diff --git a/pythonFiles/release/jedi/_compatibility.py b/pythonFiles/jedi/_compatibility.py old mode 100755 new mode 100644 similarity index 100% rename from pythonFiles/release/jedi/_compatibility.py rename to pythonFiles/jedi/_compatibility.py diff --git a/pythonFiles/release/jedi/api/__init__.py b/pythonFiles/jedi/api/__init__.py old mode 100755 new mode 100644 similarity index 100% rename from pythonFiles/release/jedi/api/__init__.py rename to pythonFiles/jedi/api/__init__.py diff --git a/pythonFiles/release/jedi/api/classes.py b/pythonFiles/jedi/api/classes.py old mode 100755 new mode 100644 similarity index 100% rename from pythonFiles/release/jedi/api/classes.py rename to pythonFiles/jedi/api/classes.py diff --git a/pythonFiles/release/jedi/api/completion.py b/pythonFiles/jedi/api/completion.py similarity index 100% rename from pythonFiles/release/jedi/api/completion.py rename to pythonFiles/jedi/api/completion.py diff --git a/pythonFiles/release/jedi/api/helpers.py b/pythonFiles/jedi/api/helpers.py old mode 100755 new mode 100644 similarity index 100% rename from pythonFiles/release/jedi/api/helpers.py rename to pythonFiles/jedi/api/helpers.py diff --git a/pythonFiles/release/jedi/api/interpreter.py b/pythonFiles/jedi/api/interpreter.py old mode 100755 new mode 100644 similarity index 100% rename from pythonFiles/release/jedi/api/interpreter.py rename to pythonFiles/jedi/api/interpreter.py diff --git a/pythonFiles/release/jedi/api/keywords.py b/pythonFiles/jedi/api/keywords.py old mode 100755 new mode 100644 similarity index 100% rename from pythonFiles/release/jedi/api/keywords.py rename to pythonFiles/jedi/api/keywords.py diff --git a/pythonFiles/preview/jedi/api/replstartup.py b/pythonFiles/jedi/api/replstartup.py similarity index 100% rename from pythonFiles/preview/jedi/api/replstartup.py rename to pythonFiles/jedi/api/replstartup.py diff --git a/pythonFiles/release/jedi/cache.py b/pythonFiles/jedi/cache.py old mode 100755 new mode 100644 similarity index 100% rename from pythonFiles/release/jedi/cache.py rename to pythonFiles/jedi/cache.py diff --git a/pythonFiles/release/jedi/common/__init__.py b/pythonFiles/jedi/common/__init__.py similarity index 100% rename from pythonFiles/release/jedi/common/__init__.py rename to pythonFiles/jedi/common/__init__.py diff --git a/pythonFiles/release/jedi/common/context.py b/pythonFiles/jedi/common/context.py similarity index 100% rename from pythonFiles/release/jedi/common/context.py rename to pythonFiles/jedi/common/context.py diff --git a/pythonFiles/release/jedi/debug.py b/pythonFiles/jedi/debug.py old mode 100755 new mode 100644 similarity index 100% rename from pythonFiles/release/jedi/debug.py rename to pythonFiles/jedi/debug.py diff --git a/pythonFiles/release/jedi/evaluate/__init__.py b/pythonFiles/jedi/evaluate/__init__.py old mode 100755 new mode 100644 similarity index 100% rename from pythonFiles/release/jedi/evaluate/__init__.py rename to pythonFiles/jedi/evaluate/__init__.py diff --git a/pythonFiles/release/jedi/evaluate/analysis.py b/pythonFiles/jedi/evaluate/analysis.py old mode 100755 new mode 100644 similarity index 100% rename from pythonFiles/release/jedi/evaluate/analysis.py rename to pythonFiles/jedi/evaluate/analysis.py diff --git a/pythonFiles/release/jedi/evaluate/arguments.py b/pythonFiles/jedi/evaluate/arguments.py similarity index 100% rename from pythonFiles/release/jedi/evaluate/arguments.py rename to pythonFiles/jedi/evaluate/arguments.py diff --git a/pythonFiles/release/jedi/evaluate/base_context.py b/pythonFiles/jedi/evaluate/base_context.py similarity index 100% rename from pythonFiles/release/jedi/evaluate/base_context.py rename to pythonFiles/jedi/evaluate/base_context.py diff --git a/pythonFiles/release/jedi/evaluate/cache.py b/pythonFiles/jedi/evaluate/cache.py old mode 100755 new mode 100644 similarity index 100% rename from pythonFiles/release/jedi/evaluate/cache.py rename to pythonFiles/jedi/evaluate/cache.py diff --git a/pythonFiles/release/jedi/evaluate/compiled/__init__.py b/pythonFiles/jedi/evaluate/compiled/__init__.py old mode 100755 new mode 100644 similarity index 100% rename from pythonFiles/release/jedi/evaluate/compiled/__init__.py rename to pythonFiles/jedi/evaluate/compiled/__init__.py diff --git a/pythonFiles/release/jedi/evaluate/compiled/fake.py b/pythonFiles/jedi/evaluate/compiled/fake.py old mode 100755 new mode 100644 similarity index 100% rename from pythonFiles/release/jedi/evaluate/compiled/fake.py rename to pythonFiles/jedi/evaluate/compiled/fake.py diff --git a/pythonFiles/preview/jedi/evaluate/compiled/fake/_functools.pym b/pythonFiles/jedi/evaluate/compiled/fake/_functools.pym similarity index 100% rename from pythonFiles/preview/jedi/evaluate/compiled/fake/_functools.pym rename to pythonFiles/jedi/evaluate/compiled/fake/_functools.pym diff --git a/pythonFiles/preview/jedi/evaluate/compiled/fake/_sqlite3.pym b/pythonFiles/jedi/evaluate/compiled/fake/_sqlite3.pym similarity index 100% rename from pythonFiles/preview/jedi/evaluate/compiled/fake/_sqlite3.pym rename to pythonFiles/jedi/evaluate/compiled/fake/_sqlite3.pym diff --git a/pythonFiles/preview/jedi/evaluate/compiled/fake/_sre.pym b/pythonFiles/jedi/evaluate/compiled/fake/_sre.pym similarity index 100% rename from pythonFiles/preview/jedi/evaluate/compiled/fake/_sre.pym rename to pythonFiles/jedi/evaluate/compiled/fake/_sre.pym diff --git a/pythonFiles/release/jedi/evaluate/compiled/fake/_weakref.pym b/pythonFiles/jedi/evaluate/compiled/fake/_weakref.pym old mode 100755 new mode 100644 similarity index 100% rename from pythonFiles/release/jedi/evaluate/compiled/fake/_weakref.pym rename to pythonFiles/jedi/evaluate/compiled/fake/_weakref.pym diff --git a/pythonFiles/release/jedi/evaluate/compiled/fake/builtins.pym b/pythonFiles/jedi/evaluate/compiled/fake/builtins.pym old mode 100755 new mode 100644 similarity index 100% rename from pythonFiles/release/jedi/evaluate/compiled/fake/builtins.pym rename to pythonFiles/jedi/evaluate/compiled/fake/builtins.pym diff --git a/pythonFiles/preview/jedi/evaluate/compiled/fake/datetime.pym b/pythonFiles/jedi/evaluate/compiled/fake/datetime.pym similarity index 100% rename from pythonFiles/preview/jedi/evaluate/compiled/fake/datetime.pym rename to pythonFiles/jedi/evaluate/compiled/fake/datetime.pym diff --git a/pythonFiles/release/jedi/evaluate/compiled/fake/io.pym b/pythonFiles/jedi/evaluate/compiled/fake/io.pym old mode 100755 new mode 100644 similarity index 100% rename from pythonFiles/release/jedi/evaluate/compiled/fake/io.pym rename to pythonFiles/jedi/evaluate/compiled/fake/io.pym diff --git a/pythonFiles/release/jedi/evaluate/compiled/fake/operator.pym b/pythonFiles/jedi/evaluate/compiled/fake/operator.pym similarity index 100% rename from pythonFiles/release/jedi/evaluate/compiled/fake/operator.pym rename to pythonFiles/jedi/evaluate/compiled/fake/operator.pym diff --git a/pythonFiles/preview/jedi/evaluate/compiled/fake/posix.pym b/pythonFiles/jedi/evaluate/compiled/fake/posix.pym similarity index 100% rename from pythonFiles/preview/jedi/evaluate/compiled/fake/posix.pym rename to pythonFiles/jedi/evaluate/compiled/fake/posix.pym diff --git a/pythonFiles/release/jedi/evaluate/compiled/getattr_static.py b/pythonFiles/jedi/evaluate/compiled/getattr_static.py similarity index 100% rename from pythonFiles/release/jedi/evaluate/compiled/getattr_static.py rename to pythonFiles/jedi/evaluate/compiled/getattr_static.py diff --git a/pythonFiles/release/jedi/evaluate/compiled/mixed.py b/pythonFiles/jedi/evaluate/compiled/mixed.py similarity index 100% rename from pythonFiles/release/jedi/evaluate/compiled/mixed.py rename to pythonFiles/jedi/evaluate/compiled/mixed.py diff --git a/pythonFiles/release/jedi/evaluate/context/__init__.py b/pythonFiles/jedi/evaluate/context/__init__.py similarity index 100% rename from pythonFiles/release/jedi/evaluate/context/__init__.py rename to pythonFiles/jedi/evaluate/context/__init__.py diff --git a/pythonFiles/release/jedi/evaluate/context/function.py b/pythonFiles/jedi/evaluate/context/function.py similarity index 100% rename from pythonFiles/release/jedi/evaluate/context/function.py rename to pythonFiles/jedi/evaluate/context/function.py diff --git a/pythonFiles/release/jedi/evaluate/context/instance.py b/pythonFiles/jedi/evaluate/context/instance.py similarity index 100% rename from pythonFiles/release/jedi/evaluate/context/instance.py rename to pythonFiles/jedi/evaluate/context/instance.py diff --git a/pythonFiles/release/jedi/evaluate/context/iterable.py b/pythonFiles/jedi/evaluate/context/iterable.py similarity index 100% rename from pythonFiles/release/jedi/evaluate/context/iterable.py rename to pythonFiles/jedi/evaluate/context/iterable.py diff --git a/pythonFiles/release/jedi/evaluate/context/klass.py b/pythonFiles/jedi/evaluate/context/klass.py similarity index 100% rename from pythonFiles/release/jedi/evaluate/context/klass.py rename to pythonFiles/jedi/evaluate/context/klass.py diff --git a/pythonFiles/release/jedi/evaluate/context/module.py b/pythonFiles/jedi/evaluate/context/module.py similarity index 100% rename from pythonFiles/release/jedi/evaluate/context/module.py rename to pythonFiles/jedi/evaluate/context/module.py diff --git a/pythonFiles/release/jedi/evaluate/context/namespace.py b/pythonFiles/jedi/evaluate/context/namespace.py similarity index 100% rename from pythonFiles/release/jedi/evaluate/context/namespace.py rename to pythonFiles/jedi/evaluate/context/namespace.py diff --git a/pythonFiles/release/jedi/evaluate/docstrings.py b/pythonFiles/jedi/evaluate/docstrings.py old mode 100755 new mode 100644 similarity index 100% rename from pythonFiles/release/jedi/evaluate/docstrings.py rename to pythonFiles/jedi/evaluate/docstrings.py diff --git a/pythonFiles/release/jedi/evaluate/dynamic.py b/pythonFiles/jedi/evaluate/dynamic.py old mode 100755 new mode 100644 similarity index 100% rename from pythonFiles/release/jedi/evaluate/dynamic.py rename to pythonFiles/jedi/evaluate/dynamic.py diff --git a/pythonFiles/release/jedi/evaluate/filters.py b/pythonFiles/jedi/evaluate/filters.py similarity index 100% rename from pythonFiles/release/jedi/evaluate/filters.py rename to pythonFiles/jedi/evaluate/filters.py diff --git a/pythonFiles/release/jedi/evaluate/finder.py b/pythonFiles/jedi/evaluate/finder.py old mode 100755 new mode 100644 similarity index 100% rename from pythonFiles/release/jedi/evaluate/finder.py rename to pythonFiles/jedi/evaluate/finder.py diff --git a/pythonFiles/release/jedi/evaluate/flow_analysis.py b/pythonFiles/jedi/evaluate/flow_analysis.py old mode 100755 new mode 100644 similarity index 100% rename from pythonFiles/release/jedi/evaluate/flow_analysis.py rename to pythonFiles/jedi/evaluate/flow_analysis.py diff --git a/pythonFiles/release/jedi/evaluate/helpers.py b/pythonFiles/jedi/evaluate/helpers.py old mode 100755 new mode 100644 similarity index 100% rename from pythonFiles/release/jedi/evaluate/helpers.py rename to pythonFiles/jedi/evaluate/helpers.py diff --git a/pythonFiles/release/jedi/evaluate/imports.py b/pythonFiles/jedi/evaluate/imports.py old mode 100755 new mode 100644 similarity index 100% rename from pythonFiles/release/jedi/evaluate/imports.py rename to pythonFiles/jedi/evaluate/imports.py diff --git a/pythonFiles/preview/jedi/evaluate/jedi_typing.py b/pythonFiles/jedi/evaluate/jedi_typing.py similarity index 100% rename from pythonFiles/preview/jedi/evaluate/jedi_typing.py rename to pythonFiles/jedi/evaluate/jedi_typing.py diff --git a/pythonFiles/release/jedi/evaluate/lazy_context.py b/pythonFiles/jedi/evaluate/lazy_context.py similarity index 100% rename from pythonFiles/release/jedi/evaluate/lazy_context.py rename to pythonFiles/jedi/evaluate/lazy_context.py diff --git a/pythonFiles/release/jedi/evaluate/param.py b/pythonFiles/jedi/evaluate/param.py old mode 100755 new mode 100644 similarity index 100% rename from pythonFiles/release/jedi/evaluate/param.py rename to pythonFiles/jedi/evaluate/param.py diff --git a/pythonFiles/release/jedi/evaluate/parser_cache.py b/pythonFiles/jedi/evaluate/parser_cache.py similarity index 100% rename from pythonFiles/release/jedi/evaluate/parser_cache.py rename to pythonFiles/jedi/evaluate/parser_cache.py diff --git a/pythonFiles/release/jedi/evaluate/pep0484.py b/pythonFiles/jedi/evaluate/pep0484.py similarity index 100% rename from pythonFiles/release/jedi/evaluate/pep0484.py rename to pythonFiles/jedi/evaluate/pep0484.py diff --git a/pythonFiles/release/jedi/evaluate/project.py b/pythonFiles/jedi/evaluate/project.py similarity index 100% rename from pythonFiles/release/jedi/evaluate/project.py rename to pythonFiles/jedi/evaluate/project.py diff --git a/pythonFiles/release/jedi/evaluate/recursion.py b/pythonFiles/jedi/evaluate/recursion.py old mode 100755 new mode 100644 similarity index 100% rename from pythonFiles/release/jedi/evaluate/recursion.py rename to pythonFiles/jedi/evaluate/recursion.py diff --git a/pythonFiles/preview/jedi/evaluate/site.py b/pythonFiles/jedi/evaluate/site.py similarity index 100% rename from pythonFiles/preview/jedi/evaluate/site.py rename to pythonFiles/jedi/evaluate/site.py diff --git a/pythonFiles/release/jedi/evaluate/stdlib.py b/pythonFiles/jedi/evaluate/stdlib.py old mode 100755 new mode 100644 similarity index 100% rename from pythonFiles/release/jedi/evaluate/stdlib.py rename to pythonFiles/jedi/evaluate/stdlib.py diff --git a/pythonFiles/release/jedi/evaluate/syntax_tree.py b/pythonFiles/jedi/evaluate/syntax_tree.py similarity index 100% rename from pythonFiles/release/jedi/evaluate/syntax_tree.py rename to pythonFiles/jedi/evaluate/syntax_tree.py diff --git a/pythonFiles/release/jedi/evaluate/sys_path.py b/pythonFiles/jedi/evaluate/sys_path.py old mode 100755 new mode 100644 similarity index 100% rename from pythonFiles/release/jedi/evaluate/sys_path.py rename to pythonFiles/jedi/evaluate/sys_path.py diff --git a/pythonFiles/release/jedi/evaluate/usages.py b/pythonFiles/jedi/evaluate/usages.py similarity index 100% rename from pythonFiles/release/jedi/evaluate/usages.py rename to pythonFiles/jedi/evaluate/usages.py diff --git a/pythonFiles/release/jedi/evaluate/utils.py b/pythonFiles/jedi/evaluate/utils.py similarity index 100% rename from pythonFiles/release/jedi/evaluate/utils.py rename to pythonFiles/jedi/evaluate/utils.py diff --git a/pythonFiles/release/jedi/parser_utils.py b/pythonFiles/jedi/parser_utils.py similarity index 100% rename from pythonFiles/release/jedi/parser_utils.py rename to pythonFiles/jedi/parser_utils.py diff --git a/pythonFiles/release/jedi/refactoring.py b/pythonFiles/jedi/refactoring.py old mode 100755 new mode 100644 similarity index 100% rename from pythonFiles/release/jedi/refactoring.py rename to pythonFiles/jedi/refactoring.py diff --git a/pythonFiles/release/jedi/settings.py b/pythonFiles/jedi/settings.py old mode 100755 new mode 100644 similarity index 100% rename from pythonFiles/release/jedi/settings.py rename to pythonFiles/jedi/settings.py diff --git a/pythonFiles/release/jedi/utils.py b/pythonFiles/jedi/utils.py old mode 100755 new mode 100644 similarity index 100% rename from pythonFiles/release/jedi/utils.py rename to pythonFiles/jedi/utils.py diff --git a/pythonFiles/parso/__init__.py b/pythonFiles/parso/__init__.py new file mode 100644 index 000000000000..f0a0fc4f5015 --- /dev/null +++ b/pythonFiles/parso/__init__.py @@ -0,0 +1,58 @@ +r""" +Parso is a Python parser that supports error recovery and round-trip parsing +for different Python versions (in multiple Python versions). Parso is also able +to list multiple syntax errors in your python file. + +Parso has been battle-tested by jedi_. It was pulled out of jedi to be useful +for other projects as well. + +Parso consists of a small API to parse Python and analyse the syntax tree. + +.. _jedi: https://github.com/davidhalter/jedi + +A simple example: + +>>> import parso +>>> module = parso.parse('hello + 1', version="3.6") +>>> expr = module.children[0] +>>> expr +PythonNode(arith_expr, [, , ]) +>>> print(expr.get_code()) +hello + 1 +>>> name = expr.children[0] +>>> name + +>>> name.end_pos +(1, 5) +>>> expr.end_pos +(1, 9) + +To list multiple issues: + +>>> grammar = parso.load_grammar() +>>> module = grammar.parse('foo +\nbar\ncontinue') +>>> error1, error2 = grammar.iter_errors(module) +>>> error1.message +'SyntaxError: invalid syntax' +>>> error2.message +"SyntaxError: 'continue' not properly in loop" +""" + +from parso.parser import ParserSyntaxError +from parso.grammar import Grammar, load_grammar +from parso.utils import split_lines, python_bytes_to_unicode + + +__version__ = '0.1.1' + + +def parse(code=None, **kwargs): + """ + A utility function to avoid loading grammars. + Params are documented in :py:meth:`parso.Grammar.parse`. + + :param str version: The version used by :py:func:`parso.load_grammar`. + """ + version = kwargs.pop('version', None) + grammar = load_grammar(version=version) + return grammar.parse(code, **kwargs) diff --git a/pythonFiles/parso/_compatibility.py b/pythonFiles/parso/_compatibility.py new file mode 100644 index 000000000000..9ddf23dc6786 --- /dev/null +++ b/pythonFiles/parso/_compatibility.py @@ -0,0 +1,103 @@ +""" +To ensure compatibility from Python ``2.6`` - ``3.3``, a module has been +created. Clearly there is huge need to use conforming syntax. +""" +import sys +import platform + +# Cannot use sys.version.major and minor names, because in Python 2.6 it's not +# a namedtuple. +py_version = int(str(sys.version_info[0]) + str(sys.version_info[1])) + +# unicode function +try: + unicode = unicode +except NameError: + unicode = str + +is_pypy = platform.python_implementation() == 'PyPy' + + +def use_metaclass(meta, *bases): + """ Create a class with a metaclass. """ + if not bases: + bases = (object,) + return meta("HackClass", bases, {}) + + +try: + encoding = sys.stdout.encoding + if encoding is None: + encoding = 'utf-8' +except AttributeError: + encoding = 'ascii' + + +def u(string): + """Cast to unicode DAMMIT! + Written because Python2 repr always implicitly casts to a string, so we + have to cast back to a unicode (and we now that we always deal with valid + unicode, because we check that in the beginning). + """ + if py_version >= 30: + return str(string) + + if not isinstance(string, unicode): + return unicode(str(string), 'UTF-8') + return string + + +try: + FileNotFoundError = FileNotFoundError +except NameError: + FileNotFoundError = IOError + + +def utf8_repr(func): + """ + ``__repr__`` methods in Python 2 don't allow unicode objects to be + returned. Therefore cast them to utf-8 bytes in this decorator. + """ + def wrapper(self): + result = func(self) + if isinstance(result, unicode): + return result.encode('utf-8') + else: + return result + + if py_version >= 30: + return func + else: + return wrapper + + +try: + from functools import total_ordering +except ImportError: + # Python 2.6 + def total_ordering(cls): + """Class decorator that fills in missing ordering methods""" + convert = { + '__lt__': [('__gt__', lambda self, other: not (self < other or self == other)), + ('__le__', lambda self, other: self < other or self == other), + ('__ge__', lambda self, other: not self < other)], + '__le__': [('__ge__', lambda self, other: not self <= other or self == other), + ('__lt__', lambda self, other: self <= other and not self == other), + ('__gt__', lambda self, other: not self <= other)], + '__gt__': [('__lt__', lambda self, other: not (self > other or self == other)), + ('__ge__', lambda self, other: self > other or self == other), + ('__le__', lambda self, other: not self > other)], + '__ge__': [('__le__', lambda self, other: (not self >= other) or self == other), + ('__gt__', lambda self, other: self >= other and not self == other), + ('__lt__', lambda self, other: not self >= other)] + } + roots = set(dir(cls)) & set(convert) + if not roots: + raise ValueError('must define at least one ordering operation: < > <= >=') + root = max(roots) # prefer __lt__ to __le__ to __gt__ to __ge__ + for opname, opfunc in convert[root]: + if opname not in roots: + opfunc.__name__ = opname + opfunc.__doc__ = getattr(int, opname).__doc__ + setattr(cls, opname, opfunc) + return cls diff --git a/pythonFiles/parso/cache.py b/pythonFiles/parso/cache.py new file mode 100644 index 000000000000..d0465d023086 --- /dev/null +++ b/pythonFiles/parso/cache.py @@ -0,0 +1,162 @@ +import time +import os +import sys +import hashlib +import gc +import shutil +import platform +import errno +import logging + +try: + import cPickle as pickle +except: + import pickle + +from parso._compatibility import FileNotFoundError + +LOG = logging.getLogger(__name__) + + +_PICKLE_VERSION = 30 +""" +Version number (integer) for file system cache. + +Increment this number when there are any incompatible changes in +the parser tree classes. For example, the following changes +are regarded as incompatible. + +- A class name is changed. +- A class is moved to another module. +- A __slot__ of a class is changed. +""" + +_VERSION_TAG = '%s-%s%s-%s' % ( + platform.python_implementation(), + sys.version_info[0], + sys.version_info[1], + _PICKLE_VERSION +) +""" +Short name for distinguish Python implementations and versions. + +It's like `sys.implementation.cache_tag` but for Python < 3.3 +we generate something similar. See: +http://docs.python.org/3/library/sys.html#sys.implementation +""" + +def _get_default_cache_path(): + if platform.system().lower() == 'windows': + dir_ = os.path.join(os.getenv('LOCALAPPDATA') or '~', 'Parso', 'Parso') + elif platform.system().lower() == 'darwin': + dir_ = os.path.join('~', 'Library', 'Caches', 'Parso') + else: + dir_ = os.path.join(os.getenv('XDG_CACHE_HOME') or '~/.cache', 'parso') + return os.path.expanduser(dir_) + +_default_cache_path = _get_default_cache_path() +""" +The path where the cache is stored. + +On Linux, this defaults to ``~/.cache/parso/``, on OS X to +``~/Library/Caches/Parso/`` and on Windows to ``%LOCALAPPDATA%\\Parso\\Parso\\``. +On Linux, if environment variable ``$XDG_CACHE_HOME`` is set, +``$XDG_CACHE_HOME/parso`` is used instead of the default one. +""" + +parser_cache = {} + + +class _NodeCacheItem(object): + def __init__(self, node, lines, change_time=None): + self.node = node + self.lines = lines + if change_time is None: + change_time = time.time() + self.change_time = change_time + + +def load_module(hashed_grammar, path, cache_path=None): + """ + Returns a module or None, if it fails. + """ + try: + p_time = os.path.getmtime(path) + except FileNotFoundError: + return None + + try: + module_cache_item = parser_cache[hashed_grammar][path] + if p_time <= module_cache_item.change_time: + return module_cache_item.node + except KeyError: + return _load_from_file_system(hashed_grammar, path, p_time, cache_path=cache_path) + + +def _load_from_file_system(hashed_grammar, path, p_time, cache_path=None): + cache_path = _get_hashed_path(hashed_grammar, path, cache_path=cache_path) + try: + try: + if p_time > os.path.getmtime(cache_path): + # Cache is outdated + return None + except OSError as e: + if e.errno == errno.ENOENT: + # In Python 2 instead of an IOError here we get an OSError. + raise FileNotFoundError + else: + raise + + with open(cache_path, 'rb') as f: + gc.disable() + try: + module_cache_item = pickle.load(f) + finally: + gc.enable() + except FileNotFoundError: + return None + else: + parser_cache.setdefault(hashed_grammar, {})[path] = module_cache_item + LOG.debug('pickle loaded: %s', path) + return module_cache_item.node + + +def save_module(hashed_grammar, path, module, lines, pickling=True, cache_path=None): + try: + p_time = None if path is None else os.path.getmtime(path) + except OSError: + p_time = None + pickling = False + + item = _NodeCacheItem(module, lines, p_time) + parser_cache.setdefault(hashed_grammar, {})[path] = item + if pickling and path is not None: + _save_to_file_system(hashed_grammar, path, item, cache_path=cache_path) + + +def _save_to_file_system(hashed_grammar, path, item, cache_path=None): + with open(_get_hashed_path(hashed_grammar, path, cache_path=cache_path), 'wb') as f: + pickle.dump(item, f, pickle.HIGHEST_PROTOCOL) + + +def clear_cache(cache_path=None): + if cache_path is None: + cache_path = _default_cache_path + shutil.rmtree(cache_path) + parser_cache.clear() + + +def _get_hashed_path(hashed_grammar, path, cache_path=None): + directory = _get_cache_directory_path(cache_path=cache_path) + + file_hash = hashlib.sha256(path.encode("utf-8")).hexdigest() + return os.path.join(directory, '%s-%s.pkl' % (hashed_grammar, file_hash)) + + +def _get_cache_directory_path(cache_path=None): + if cache_path is None: + cache_path = _default_cache_path + directory = os.path.join(cache_path, _VERSION_TAG) + if not os.path.exists(directory): + os.makedirs(directory) + return directory diff --git a/pythonFiles/parso/grammar.py b/pythonFiles/parso/grammar.py new file mode 100644 index 000000000000..2cf26d77fb27 --- /dev/null +++ b/pythonFiles/parso/grammar.py @@ -0,0 +1,283 @@ +import hashlib +import os + +from parso._compatibility import FileNotFoundError, is_pypy +from parso.pgen2.pgen import generate_grammar +from parso.utils import split_lines, python_bytes_to_unicode, parse_version_string +from parso.python.diff import DiffParser +from parso.python.tokenize import tokenize_lines, tokenize +from parso.python import token +from parso.cache import parser_cache, load_module, save_module +from parso.parser import BaseParser +from parso.python.parser import Parser as PythonParser +from parso.python.errors import ErrorFinderConfig +from parso.python import pep8 +from parso.python import fstring + +_loaded_grammars = {} + + +class Grammar(object): + """ + :py:func:`parso.load_grammar` returns instances of this class. + + Creating custom grammars by calling this is not supported, yet. + """ + #:param text: A BNF representation of your grammar. + _error_normalizer_config = None + _token_namespace = None + _default_normalizer_config = pep8.PEP8NormalizerConfig() + + def __init__(self, text, tokenizer, parser=BaseParser, diff_parser=None): + self._pgen_grammar = generate_grammar( + text, + token_namespace=self._get_token_namespace() + ) + self._parser = parser + self._tokenizer = tokenizer + self._diff_parser = diff_parser + self._hashed = hashlib.sha256(text.encode("utf-8")).hexdigest() + + def parse(self, code=None, **kwargs): + """ + If you want to parse a Python file you want to start here, most likely. + + If you need finer grained control over the parsed instance, there will be + other ways to access it. + + :param str code: A unicode or bytes string. When it's not possible to + decode bytes to a string, returns a + :py:class:`UnicodeDecodeError`. + :param bool error_recovery: If enabled, any code will be returned. If + it is invalid, it will be returned as an error node. If disabled, + you will get a ParseError when encountering syntax errors in your + code. + :param str start_symbol: The grammar symbol that you want to parse. Only + allowed to be used when error_recovery is False. + :param str path: The path to the file you want to open. Only needed for caching. + :param bool cache: Keeps a copy of the parser tree in RAM and on disk + if a path is given. Returns the cached trees if the corresponding + files on disk have not changed. + :param bool diff_cache: Diffs the cached python module against the new + code and tries to parse only the parts that have changed. Returns + the same (changed) module that is found in cache. Using this option + requires you to not do anything anymore with the cached modules + under that path, because the contents of it might change. This + option is still somewhat experimental. If you want stability, + please don't use it. + :param bool cache_path: If given saves the parso cache in this + directory. If not given, defaults to the default cache places on + each platform. + + :return: A subclass of :py:class:`parso.tree.NodeOrLeaf`. Typically a + :py:class:`parso.python.tree.Module`. + """ + if 'start_pos' in kwargs: + raise TypeError("parse() got an unexpected keyworda argument.") + return self._parse(code=code, **kwargs) + + def _parse(self, code=None, error_recovery=True, path=None, + start_symbol=None, cache=False, diff_cache=False, + cache_path=None, start_pos=(1, 0)): + """ + Wanted python3.5 * operator and keyword only arguments. Therefore just + wrap it all. + start_pos here is just a parameter internally used. Might be public + sometime in the future. + """ + if code is None and path is None: + raise TypeError("Please provide either code or a path.") + + if start_symbol is None: + start_symbol = self._start_symbol + + if error_recovery and start_symbol != 'file_input': + raise NotImplementedError("This is currently not implemented.") + + if cache and path is not None: + module_node = load_module(self._hashed, path, cache_path=cache_path) + if module_node is not None: + return module_node + + if code is None: + with open(path, 'rb') as f: + code = f.read() + + code = python_bytes_to_unicode(code) + + lines = split_lines(code, keepends=True) + if diff_cache: + if self._diff_parser is None: + raise TypeError("You have to define a diff parser to be able " + "to use this option.") + try: + module_cache_item = parser_cache[self._hashed][path] + except KeyError: + pass + else: + module_node = module_cache_item.node + old_lines = module_cache_item.lines + if old_lines == lines: + return module_node + + new_node = self._diff_parser( + self._pgen_grammar, self._tokenizer, module_node + ).update( + old_lines=old_lines, + new_lines=lines + ) + save_module(self._hashed, path, new_node, lines, + # Never pickle in pypy, it's slow as hell. + pickling=cache and not is_pypy, + cache_path=cache_path) + return new_node + + tokens = self._tokenizer(lines, start_pos) + + p = self._parser( + self._pgen_grammar, + error_recovery=error_recovery, + start_symbol=start_symbol + ) + root_node = p.parse(tokens=tokens) + + if cache or diff_cache: + save_module(self._hashed, path, root_node, lines, + # Never pickle in pypy, it's slow as hell. + pickling=cache and not is_pypy, + cache_path=cache_path) + return root_node + + def _get_token_namespace(self): + ns = self._token_namespace + if ns is None: + raise ValueError("The token namespace should be set.") + return ns + + def iter_errors(self, node): + """ + Given a :py:class:`parso.tree.NodeOrLeaf` returns a generator of + :py:class:`parso.normalizer.Issue` objects. For Python this is + a list of syntax/indentation errors. + """ + if self._error_normalizer_config is None: + raise ValueError("No error normalizer specified for this grammar.") + + return self._get_normalizer_issues(node, self._error_normalizer_config) + + def _get_normalizer(self, normalizer_config): + if normalizer_config is None: + normalizer_config = self._default_normalizer_config + if normalizer_config is None: + raise ValueError("You need to specify a normalizer, because " + "there's no default normalizer for this tree.") + return normalizer_config.create_normalizer(self) + + def _normalize(self, node, normalizer_config=None): + """ + TODO this is not public, yet. + The returned code will be normalized, e.g. PEP8 for Python. + """ + normalizer = self._get_normalizer(normalizer_config) + return normalizer.walk(node) + + def _get_normalizer_issues(self, node, normalizer_config=None): + normalizer = self._get_normalizer(normalizer_config) + normalizer.walk(node) + return normalizer.issues + + + def __repr__(self): + labels = self._pgen_grammar.number2symbol.values() + txt = ' '.join(list(labels)[:3]) + ' ...' + return '<%s:%s>' % (self.__class__.__name__, txt) + + +class PythonGrammar(Grammar): + _error_normalizer_config = ErrorFinderConfig() + _token_namespace = token + _start_symbol = 'file_input' + + def __init__(self, version_info, bnf_text): + super(PythonGrammar, self).__init__( + bnf_text, + tokenizer=self._tokenize_lines, + parser=PythonParser, + diff_parser=DiffParser + ) + self.version_info = version_info + + def _tokenize_lines(self, lines, start_pos): + return tokenize_lines(lines, self.version_info, start_pos=start_pos) + + def _tokenize(self, code): + # Used by Jedi. + return tokenize(code, self.version_info) + + +class PythonFStringGrammar(Grammar): + _token_namespace = fstring.TokenNamespace + _start_symbol = 'fstring' + + def __init__(self): + super(PythonFStringGrammar, self).__init__( + text=fstring.GRAMMAR, + tokenizer=fstring.tokenize, + parser=fstring.Parser + ) + + def parse(self, code, **kwargs): + return self._parse(code, **kwargs) + + def _parse(self, code, error_recovery=True, start_pos=(1, 0)): + tokens = self._tokenizer(code, start_pos=start_pos) + p = self._parser( + self._pgen_grammar, + error_recovery=error_recovery, + start_symbol=self._start_symbol, + ) + return p.parse(tokens=tokens) + + def parse_leaf(self, leaf, error_recovery=True): + code = leaf._get_payload() + return self.parse(code, error_recovery=True, start_pos=leaf.start_pos) + + +def load_grammar(**kwargs): + """ + Loads a :py:class:`parso.Grammar`. The default version is the current Python + version. + + :param str version: A python version string, e.g. ``version='3.3'``. + """ + def load_grammar(language='python', version=None): + if language == 'python': + version_info = parse_version_string(version) + + file = os.path.join( + 'python', + 'grammar%s%s.txt' % (version_info.major, version_info.minor) + ) + + global _loaded_grammars + path = os.path.join(os.path.dirname(__file__), file) + try: + return _loaded_grammars[path] + except KeyError: + try: + with open(path) as f: + bnf_text = f.read() + + grammar = PythonGrammar(version_info, bnf_text) + return _loaded_grammars.setdefault(path, grammar) + except FileNotFoundError: + message = "Python version %s is currently not supported." % version + raise NotImplementedError(message) + elif language == 'python-f-string': + if version is not None: + raise NotImplementedError("Currently different versions are not supported.") + return PythonFStringGrammar() + else: + raise NotImplementedError("No support for language %s." % language) + + return load_grammar(**kwargs) diff --git a/pythonFiles/parso/normalizer.py b/pythonFiles/parso/normalizer.py new file mode 100644 index 000000000000..9a3e82e24c87 --- /dev/null +++ b/pythonFiles/parso/normalizer.py @@ -0,0 +1,184 @@ +from contextlib import contextmanager + +from parso._compatibility import use_metaclass + + +class _NormalizerMeta(type): + def __new__(cls, name, bases, dct): + new_cls = type.__new__(cls, name, bases, dct) + new_cls.rule_value_classes = {} + new_cls.rule_type_classes = {} + return new_cls + + +class Normalizer(use_metaclass(_NormalizerMeta)): + def __init__(self, grammar, config): + self.grammar = grammar + self._config = config + self.issues = [] + + self._rule_type_instances = self._instantiate_rules('rule_type_classes') + self._rule_value_instances = self._instantiate_rules('rule_value_classes') + + def _instantiate_rules(self, attr): + dct = {} + for base in type(self).mro(): + rules_map = getattr(base, attr, {}) + for type_, rule_classes in rules_map.items(): + new = [rule_cls(self) for rule_cls in rule_classes] + dct.setdefault(type_, []).extend(new) + return dct + + def walk(self, node): + self.initialize(node) + value = self.visit(node) + self.finalize() + return value + + def visit(self, node): + try: + children = node.children + except AttributeError: + return self.visit_leaf(node) + else: + with self.visit_node(node): + return ''.join(self.visit(child) for child in children) + + @contextmanager + def visit_node(self, node): + self._check_type_rules(node) + yield + + def _check_type_rules(self, node): + for rule in self._rule_type_instances.get(node.type, []): + rule.feed_node(node) + + def visit_leaf(self, leaf): + self._check_type_rules(leaf) + + for rule in self._rule_value_instances.get(leaf.value, []): + rule.feed_node(leaf) + + return leaf.prefix + leaf.value + + def initialize(self, node): + pass + + def finalize(self): + pass + + def add_issue(self, node, code, message): + issue = Issue(node, code, message) + if issue not in self.issues: + self.issues.append(issue) + return True + + @classmethod + def register_rule(cls, **kwargs): + """ + Use it as a class decorator:: + + normalizer = Normalizer('grammar', 'config') + @normalizer.register_rule(value='foo') + class MyRule(Rule): + error_code = 42 + """ + return cls._register_rule(**kwargs) + + @classmethod + def _register_rule(cls, value=None, values=(), type=None, types=()): + values = list(values) + types = list(types) + if value is not None: + values.append(value) + if type is not None: + types.append(type) + + if not values and not types: + raise ValueError("You must register at least something.") + + def decorator(rule_cls): + for v in values: + cls.rule_value_classes.setdefault(v, []).append(rule_cls) + for t in types: + cls.rule_type_classes.setdefault(t, []).append(rule_cls) + return rule_cls + + return decorator + + +class NormalizerConfig(object): + normalizer_class = Normalizer + + def create_normalizer(self, grammar): + if self.normalizer_class is None: + return None + + return self.normalizer_class(grammar, self) + + +class Issue(object): + def __init__(self, node, code, message): + self._node = node + self.code = code + """ + An integer code that stands for the type of error. + """ + self.message = message + """ + A message (string) for the issue. + """ + self.start_pos = node.start_pos + """ + The start position position of the error as a tuple (line, column). As + always in |parso| the first line is 1 and the first column 0. + """ + + def __eq__(self, other): + return self.start_pos == other.start_pos and self.code == other.code + + def __ne__(self, other): + return not self.__eq__(other) + + def __hash__(self): + return hash((self.code, self.start_pos)) + + def __repr__(self): + return '<%s: %s>' % (self.__class__.__name__, self.code) + + + +class Rule(object): + code = None + message = None + + def __init__(self, normalizer): + self._normalizer = normalizer + + def is_issue(self, node): + raise NotImplementedError() + + def get_node(self, node): + return node + + def _get_message(self, message): + if message is None: + message = self.message + if message is None: + raise ValueError("The message on the class is not set.") + return message + + def add_issue(self, node, code=None, message=None): + if code is None: + code = self.code + if code is None: + raise ValueError("The error code on the class is not set.") + + message = self._get_message(message) + + self._normalizer.add_issue(node, code, message) + + def feed_node(self, node): + if self.is_issue(node): + issue_node = self.get_node(node) + self.add_issue(issue_node) diff --git a/pythonFiles/parso/parser.py b/pythonFiles/parso/parser.py new file mode 100644 index 000000000000..555ebc712f73 --- /dev/null +++ b/pythonFiles/parso/parser.py @@ -0,0 +1,78 @@ +""" +The ``Parser`` tries to convert the available Python code in an easy to read +format, something like an abstract syntax tree. The classes who represent this +tree, are sitting in the :mod:`parso.tree` module. + +The Python module ``tokenize`` is a very important part in the ``Parser``, +because it splits the code into different words (tokens). Sometimes it looks a +bit messy. Sorry for that! You might ask now: "Why didn't you use the ``ast`` +module for this? Well, ``ast`` does a very good job understanding proper Python +code, but fails to work as soon as there's a single line of broken code. + +There's one important optimization that needs to be known: Statements are not +being parsed completely. ``Statement`` is just a representation of the tokens +within the statement. This lowers memory usage and cpu time and reduces the +complexity of the ``Parser`` (there's another parser sitting inside +``Statement``, which produces ``Array`` and ``Call``). +""" +from parso import tree +from parso.pgen2.parse import PgenParser + + +class ParserSyntaxError(Exception): + """ + Contains error information about the parser tree. + + May be raised as an exception. + """ + def __init__(self, message, error_leaf): + self.message = message + self.error_leaf = error_leaf + + +class BaseParser(object): + node_map = {} + default_node = tree.Node + + leaf_map = { + } + default_leaf = tree.Leaf + + def __init__(self, pgen_grammar, start_symbol='file_input', error_recovery=False): + self._pgen_grammar = pgen_grammar + self._start_symbol = start_symbol + self._error_recovery = error_recovery + + def parse(self, tokens): + start_number = self._pgen_grammar.symbol2number[self._start_symbol] + self.pgen_parser = PgenParser( + self._pgen_grammar, self.convert_node, self.convert_leaf, + self.error_recovery, start_number + ) + + node = self.pgen_parser.parse(tokens) + # The stack is empty now, we don't need it anymore. + del self.pgen_parser + return node + + def error_recovery(self, pgen_grammar, stack, arcs, typ, value, start_pos, prefix, + add_token_callback): + if self._error_recovery: + raise NotImplementedError("Error Recovery is not implemented") + else: + error_leaf = tree.ErrorLeaf('TODO %s' % typ, value, start_pos, prefix) + raise ParserSyntaxError('SyntaxError: invalid syntax', error_leaf) + + def convert_node(self, pgen_grammar, type_, children): + # TODO REMOVE symbol, we don't want type here. + symbol = pgen_grammar.number2symbol[type_] + try: + return self.node_map[symbol](children) + except KeyError: + return self.default_node(symbol, children) + + def convert_leaf(self, pgen_grammar, type_, value, prefix, start_pos): + try: + return self.leaf_map[type_](value, start_pos, prefix) + except KeyError: + return self.default_leaf(value, start_pos, prefix) diff --git a/pythonFiles/preview/jedi/parser/pgen2/__init__.py b/pythonFiles/parso/pgen2/__init__.py similarity index 100% rename from pythonFiles/preview/jedi/parser/pgen2/__init__.py rename to pythonFiles/parso/pgen2/__init__.py diff --git a/pythonFiles/preview/jedi/parser/pgen2/grammar.py b/pythonFiles/parso/pgen2/grammar.py similarity index 97% rename from pythonFiles/preview/jedi/parser/pgen2/grammar.py rename to pythonFiles/parso/pgen2/grammar.py index 414c0dbe9f01..e5f211426fad 100644 --- a/pythonFiles/preview/jedi/parser/pgen2/grammar.py +++ b/pythonFiles/parso/pgen2/grammar.py @@ -16,8 +16,10 @@ """ -# Python imports -import pickle +try: + import cPickle as pickle +except: + import pickle class Grammar(object): @@ -74,7 +76,7 @@ class Grammar(object): """ - def __init__(self): + def __init__(self, bnf_text): self.symbol2number = {} self.number2symbol = {} self.states = [] @@ -83,6 +85,7 @@ def __init__(self): self.keywords = {} self.tokens = {} self.symbol2label = {} + self.label2symbol = {} self.start = 256 def dump(self, filename): diff --git a/pythonFiles/preview/jedi/parser/pgen2/parse.py b/pythonFiles/parso/pgen2/parse.py similarity index 87% rename from pythonFiles/preview/jedi/parser/pgen2/parse.py rename to pythonFiles/parso/pgen2/parse.py index 9e74838c7c99..aaacfcebe44e 100644 --- a/pythonFiles/preview/jedi/parser/pgen2/parse.py +++ b/pythonFiles/parso/pgen2/parse.py @@ -14,8 +14,7 @@ how this parsing engine works. """ -# Local imports -from jedi.parser import tokenize +from parso.python import tokenize class InternalParseError(Exception): @@ -34,6 +33,12 @@ def __init__(self, msg, type, value, start_pos): self.start_pos = start_pos +class Stack(list): + def get_tos_nodes(self): + tos = self[-1] + return tos[2][1] + + def token_to_ilabel(grammar, type_, value): # Map from token to label if type_ == tokenize.NAME: @@ -57,7 +62,7 @@ class PgenParser(object): p = Parser(grammar, [converter]) # create instance p.setup([start]) # prepare for parsing : - if p.addtoken(...): # parse a token + if p.add_token(...): # parse a token break root = p.rootnode # root of abstract syntax tree @@ -70,7 +75,7 @@ class PgenParser(object): See driver.py for how to get input tokens by tokenizing a file or string. - Parsing is complete when addtoken() returns True; the root of the + Parsing is complete when add_token() returns True; the root of the abstract syntax tree can then be retrieved from the rootnode instance variable. When a syntax error occurs, error_recovery() is called. There is no error recovery; the parser cannot be used @@ -114,13 +119,13 @@ def __init__(self, grammar, convert_node, convert_leaf, error_recovery, start): # where children is a list of nodes or None newnode = (start, []) stackentry = (self.grammar.dfas[start], 0, newnode) - self.stack = [stackentry] + self.stack = Stack([stackentry]) self.rootnode = None self.error_recovery = error_recovery - def parse(self, tokenizer): - for type_, value, start_pos, prefix in tokenizer: - if self.addtoken(type_, value, start_pos, prefix): + def parse(self, tokens): + for type_, value, start_pos, prefix in tokens: + if self.add_token(type_, value, start_pos, prefix): break else: # We never broke out -- EOF is too soon -- Unfinished statement. @@ -130,27 +135,32 @@ def parse(self, tokenizer): raise InternalParseError("incomplete input", type_, value, start_pos) return self.rootnode - def addtoken(self, type_, value, start_pos, prefix): + def add_token(self, type_, value, start_pos, prefix): """Add a token; return True if this is the end of the program.""" ilabel = token_to_ilabel(self.grammar, type_, value) # Loop until the token is shifted; may raise exceptions + _gram = self.grammar + _labels = _gram.labels + _push = self._push + _pop = self._pop + _shift = self._shift while True: dfa, state, node = self.stack[-1] states, first = dfa arcs = states[state] # Look for a state with this label for i, newstate in arcs: - t, v = self.grammar.labels[i] + t, v = _labels[i] if ilabel == i: # Look it up in the list of labels assert t < 256 # Shift a token; we're done with it - self.shift(type_, value, newstate, prefix, start_pos) + _shift(type_, value, newstate, prefix, start_pos) # Pop while we are in an accept-only state state = newstate while states[state] == [(0, state)]: - self.pop() + _pop() if not self.stack: # Done parsing! return True @@ -160,39 +170,39 @@ def addtoken(self, type_, value, start_pos, prefix): return False elif t >= 256: # See if it's a symbol and if we're in its first set - itsdfa = self.grammar.dfas[t] + itsdfa = _gram.dfas[t] itsstates, itsfirst = itsdfa if ilabel in itsfirst: # Push a symbol - self.push(t, itsdfa, newstate) + _push(t, itsdfa, newstate) break # To continue the outer while loop else: if (0, state) in arcs: # An accepting state, pop it and try something else - self.pop() + _pop() if not self.stack: # Done parsing, but another token is input raise InternalParseError("too much input", type_, value, start_pos) else: self.error_recovery(self.grammar, self.stack, arcs, type_, - value, start_pos, prefix, self.addtoken) + value, start_pos, prefix, self.add_token) break - def shift(self, type_, value, newstate, prefix, start_pos): + def _shift(self, type_, value, newstate, prefix, start_pos): """Shift a token. (Internal)""" dfa, state, node = self.stack[-1] newnode = self.convert_leaf(self.grammar, type_, value, prefix, start_pos) node[-1].append(newnode) self.stack[-1] = (dfa, newstate, node) - def push(self, type_, newdfa, newstate): + def _push(self, type_, newdfa, newstate): """Push a nonterminal. (Internal)""" dfa, state, node = self.stack[-1] newnode = (type_, []) self.stack[-1] = (dfa, newstate, node) self.stack.append((newdfa, 0, newnode)) - def pop(self): + def _pop(self): """Pop a nonterminal. (Internal)""" popdfa, popstate, (type_, children) = self.stack.pop() # If there's exactly one child, return that child instead of creating a diff --git a/pythonFiles/release/jedi/parser/pgen2/pgen.py b/pythonFiles/parso/pgen2/pgen.py old mode 100755 new mode 100644 similarity index 76% rename from pythonFiles/release/jedi/parser/pgen2/pgen.py rename to pythonFiles/parso/pgen2/pgen.py index fa2742dd5dc4..10ef6ffd1532 --- a/pythonFiles/release/jedi/parser/pgen2/pgen.py +++ b/pythonFiles/parso/pgen2/pgen.py @@ -5,30 +5,27 @@ # Copyright 2014 David Halter. Integration into Jedi. # Modifications are dual-licensed: MIT and PSF. -# Pgen imports -from . import grammar -from jedi.parser import token -from jedi.parser import tokenize +from parso.pgen2 import grammar +from parso.python import token +from parso.python import tokenize +from parso.utils import parse_version_string class ParserGenerator(object): - def __init__(self, filename, stream=None): - close_stream = None - if stream is None: - stream = open(filename) - close_stream = stream.close - self.filename = filename - self.stream = stream - self.generator = tokenize.generate_tokens(stream.readline) - self.gettoken() # Initialize lookahead - self.dfas, self.startsymbol = self.parse() - if close_stream is not None: - close_stream() + def __init__(self, bnf_text, token_namespace): + self._bnf_text = bnf_text + self.generator = tokenize.tokenize( + bnf_text, + version_info=parse_version_string('3.6') + ) + self._gettoken() # Initialize lookahead + self.dfas, self.startsymbol = self._parse() self.first = {} # map from symbol name to set of tokens - self.addfirstsets() + self._addfirstsets() + self._token_namespace = token_namespace def make_grammar(self): - c = grammar.Grammar() + c = grammar.Grammar(self._bnf_text) names = list(self.dfas.keys()) names.sort() names.remove(self.startsymbol) @@ -43,25 +40,25 @@ def make_grammar(self): for state in dfa: arcs = [] for label, next in state.arcs.items(): - arcs.append((self.make_label(c, label), dfa.index(next))) + arcs.append((self._make_label(c, label), dfa.index(next))) if state.isfinal: arcs.append((0, dfa.index(state))) states.append(arcs) c.states.append(states) - c.dfas[c.symbol2number[name]] = (states, self.make_first(c, name)) + c.dfas[c.symbol2number[name]] = (states, self._make_first(c, name)) c.start = c.symbol2number[self.startsymbol] return c - def make_first(self, c, name): + def _make_first(self, c, name): rawfirst = self.first[name] first = {} for label in rawfirst: - ilabel = self.make_label(c, label) + ilabel = self._make_label(c, label) ##assert ilabel not in first # XXX failed on <> ... != first[ilabel] = 1 return first - def make_label(self, c, label): + def _make_label(self, c, label): # XXX Maybe this should be a method on a subclass of converter? ilabel = len(c.labels) if label[0].isalpha(): @@ -73,12 +70,12 @@ def make_label(self, c, label): else: c.labels.append((c.symbol2number[label], None)) c.symbol2label[label] = ilabel + c.label2symbol[ilabel] = label return ilabel else: # A named token (NAME, NUMBER, STRING) - itoken = getattr(token, label, None) + itoken = getattr(self._token_namespace, label, None) assert isinstance(itoken, int), label - assert itoken in token.tok_name, label if itoken in c.tokens: return c.tokens[itoken] else: @@ -94,12 +91,13 @@ def make_label(self, c, label): if value in c.keywords: return c.keywords[value] else: + # TODO this might be an issue?! Using token.NAME here? c.labels.append((token.NAME, value)) c.keywords[value] = ilabel return ilabel else: # An operator (any non-numeric token) - itoken = token.opmap[value] # Fails if unknown token + itoken = self._token_namespace.generate_token_id(value) if itoken in c.tokens: return c.tokens[itoken] else: @@ -107,15 +105,15 @@ def make_label(self, c, label): c.tokens[itoken] = ilabel return ilabel - def addfirstsets(self): + def _addfirstsets(self): names = list(self.dfas.keys()) names.sort() for name in names: if name not in self.first: - self.calcfirst(name) + self._calcfirst(name) #print name, self.first[name].keys() - def calcfirst(self, name): + def _calcfirst(self, name): dfa = self.dfas[name] self.first[name] = None # dummy to detect left recursion state = dfa[0] @@ -128,7 +126,7 @@ def calcfirst(self, name): if fset is None: raise ValueError("recursion for rule %r" % name) else: - self.calcfirst(label) + self._calcfirst(label) fset = self.first[label] totalset.update(fset) overlapcheck[label] = fset @@ -145,23 +143,23 @@ def calcfirst(self, name): inverse[symbol] = label self.first[name] = totalset - def parse(self): + def _parse(self): dfas = {} startsymbol = None # MSTART: (NEWLINE | RULE)* ENDMARKER while self.type != token.ENDMARKER: while self.type == token.NEWLINE: - self.gettoken() + self._gettoken() # RULE: NAME ':' RHS NEWLINE - name = self.expect(token.NAME) - self.expect(token.OP, ":") - a, z = self.parse_rhs() - self.expect(token.NEWLINE) - #self.dump_nfa(name, a, z) - dfa = self.make_dfa(a, z) - #self.dump_dfa(name, dfa) + name = self._expect(token.NAME) + self._expect(token.COLON) + a, z = self._parse_rhs() + self._expect(token.NEWLINE) + #self._dump_nfa(name, a, z) + dfa = self._make_dfa(a, z) + #self._dump_dfa(name, dfa) # oldlen = len(dfa) - self.simplify_dfa(dfa) + self._simplify_dfa(dfa) # newlen = len(dfa) dfas[name] = dfa #print name, oldlen, newlen @@ -169,7 +167,7 @@ def parse(self): startsymbol = name return dfas, startsymbol - def make_dfa(self, start, finish): + def _make_dfa(self, start, finish): # To turn an NFA into a DFA, we define the states of the DFA # to correspond to *sets* of states of the NFA. Then do some # state reduction. Let's represent sets as dicts with 1 for @@ -208,7 +206,7 @@ def addclosure(state, base): state.addarc(st, label) return states # List of DFAState instances; first one is start - def dump_nfa(self, name, start, finish): + def _dump_nfa(self, name, start, finish): print("Dump of NFA for", name) todo = [start] for i, state in enumerate(todo): @@ -224,14 +222,14 @@ def dump_nfa(self, name, start, finish): else: print(" %s -> %d" % (label, j)) - def dump_dfa(self, name, dfa): + def _dump_dfa(self, name, dfa): print("Dump of DFA for", name) for i, state in enumerate(dfa): print(" State", i, state.isfinal and "(final)" or "") for label, next in state.arcs.items(): print(" %s -> %d" % (label, dfa.index(next))) - def simplify_dfa(self, dfa): + def _simplify_dfa(self, dfa): # This is not theoretically optimal, but works well enough. # Algorithm: repeatedly look for two states that have the same # set of arcs (same labels pointing to the same nodes) and @@ -252,9 +250,9 @@ def simplify_dfa(self, dfa): changes = True break - def parse_rhs(self): + def _parse_rhs(self): # RHS: ALT ('|' ALT)* - a, z = self.parse_alt() + a, z = self._parse_alt() if self.value != "|": return a, z else: @@ -263,82 +261,81 @@ def parse_rhs(self): aa.addarc(a) z.addarc(zz) while self.value == "|": - self.gettoken() - a, z = self.parse_alt() + self._gettoken() + a, z = self._parse_alt() aa.addarc(a) z.addarc(zz) return aa, zz - def parse_alt(self): + def _parse_alt(self): # ALT: ITEM+ - a, b = self.parse_item() + a, b = self._parse_item() while (self.value in ("(", "[") or self.type in (token.NAME, token.STRING)): - c, d = self.parse_item() + c, d = self._parse_item() b.addarc(c) b = d return a, b - def parse_item(self): + def _parse_item(self): # ITEM: '[' RHS ']' | ATOM ['+' | '*'] if self.value == "[": - self.gettoken() - a, z = self.parse_rhs() - self.expect(token.OP, "]") + self._gettoken() + a, z = self._parse_rhs() + self._expect(token.RSQB) a.addarc(z) return a, z else: - a, z = self.parse_atom() + a, z = self._parse_atom() value = self.value if value not in ("+", "*"): return a, z - self.gettoken() + self._gettoken() z.addarc(a) if value == "+": return a, z else: return a, a - def parse_atom(self): + def _parse_atom(self): # ATOM: '(' RHS ')' | NAME | STRING if self.value == "(": - self.gettoken() - a, z = self.parse_rhs() - self.expect(token.OP, ")") + self._gettoken() + a, z = self._parse_rhs() + self._expect(token.RPAR) return a, z elif self.type in (token.NAME, token.STRING): a = NFAState() z = NFAState() a.addarc(z, self.value) - self.gettoken() + self._gettoken() return a, z else: - self.raise_error("expected (...) or NAME or STRING, got %s/%s", - self.type, self.value) + self._raise_error("expected (...) or NAME or STRING, got %s/%s", + self.type, self.value) - def expect(self, type, value=None): - if self.type != type or (value is not None and self.value != value): - self.raise_error("expected %s/%s, got %s/%s", - type, value, self.type, self.value) + def _expect(self, type): + if self.type != type: + self._raise_error("expected %s, got %s(%s)", + type, self.type, self.value) value = self.value - self.gettoken() + self._gettoken() return value - def gettoken(self): + def _gettoken(self): tup = next(self.generator) while tup[0] in (token.COMMENT, token.NL): tup = next(self.generator) self.type, self.value, self.begin, prefix = tup - #print tokenize.tok_name[self.type], repr(self.value) - def raise_error(self, msg, *args): + def _raise_error(self, msg, *args): if args: try: msg = msg % args except: msg = " ".join([msg] + list(map(str, args))) - line = open(self.filename).readlines()[self.begin[0]] - raise SyntaxError(msg, (self.filename, self.begin[0], + line = self._bnf_text.splitlines()[self.begin[0] - 1] + raise SyntaxError(msg, ('', self.begin[0], self.begin[1], line)) @@ -389,6 +386,14 @@ def __eq__(self, other): __hash__ = None # For Py3 compatibility. -def generate_grammar(filename="Grammar.txt"): - p = ParserGenerator(filename) +def generate_grammar(bnf_text, token_namespace): + """ + ``bnf_text`` is a grammar in extended BNF (using * for repetition, + for + at-least-once repetition, [] for optional parts, | for alternatives and () + for grouping). + + It's not EBNF according to ISO/IEC 14977. It's a dialect Python uses in its + own parser. + """ + p = ParserGenerator(bnf_text, token_namespace) return p.make_grammar() diff --git a/pythonFiles/parso/python/__init__.py b/pythonFiles/parso/python/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/pythonFiles/parso/python/diff.py b/pythonFiles/parso/python/diff.py new file mode 100644 index 000000000000..c2e44fd3cb21 --- /dev/null +++ b/pythonFiles/parso/python/diff.py @@ -0,0 +1,587 @@ +""" +Basically a contains parser that is faster, because it tries to parse only +parts and if anything changes, it only reparses the changed parts. + +It works with a simple diff in the beginning and will try to reuse old parser +fragments. +""" +import re +import difflib +from collections import namedtuple +import logging + +from parso.utils import split_lines +from parso.python.parser import Parser +from parso.python.tree import EndMarker +from parso.python.tokenize import (NEWLINE, PythonToken, ERROR_DEDENT, + ENDMARKER, INDENT, DEDENT) + +LOG = logging.getLogger(__name__) + + +def _get_last_line(node_or_leaf): + last_leaf = node_or_leaf.get_last_leaf() + if _ends_with_newline(last_leaf): + return last_leaf.start_pos[0] + else: + return last_leaf.end_pos[0] + + +def _ends_with_newline(leaf, suffix=''): + if leaf.type == 'error_leaf': + typ = leaf.original_type + else: + typ = leaf.type + + return typ == 'newline' or suffix.endswith('\n') + + +def _flows_finished(pgen_grammar, stack): + """ + if, while, for and try might not be finished, because another part might + still be parsed. + """ + for dfa, newstate, (symbol_number, nodes) in stack: + if pgen_grammar.number2symbol[symbol_number] in ('if_stmt', 'while_stmt', + 'for_stmt', 'try_stmt'): + return False + return True + + +def suite_or_file_input_is_valid(pgen_grammar, stack): + if not _flows_finished(pgen_grammar, stack): + return False + + for dfa, newstate, (symbol_number, nodes) in reversed(stack): + if pgen_grammar.number2symbol[symbol_number] == 'suite': + # If only newline is in the suite, the suite is not valid, yet. + return len(nodes) > 1 + # Not reaching a suite means that we're dealing with file_input levels + # where there's no need for a valid statement in it. It can also be empty. + return True + + +def _is_flow_node(node): + try: + value = node.children[0].value + except AttributeError: + return False + return value in ('if', 'for', 'while', 'try') + + +class _PositionUpdatingFinished(Exception): + pass + + +def _update_positions(nodes, line_offset, last_leaf): + for node in nodes: + try: + children = node.children + except AttributeError: + # Is a leaf + node.line += line_offset + if node is last_leaf: + raise _PositionUpdatingFinished + else: + _update_positions(children, line_offset, last_leaf) + + +class DiffParser(object): + """ + An advanced form of parsing a file faster. Unfortunately comes with huge + side effects. It changes the given module. + """ + def __init__(self, pgen_grammar, tokenizer, module): + self._pgen_grammar = pgen_grammar + self._tokenizer = tokenizer + self._module = module + + def _reset(self): + self._copy_count = 0 + self._parser_count = 0 + + self._nodes_stack = _NodesStack(self._module) + + def update(self, old_lines, new_lines): + ''' + The algorithm works as follows: + + Equal: + - Assure that the start is a newline, otherwise parse until we get + one. + - Copy from parsed_until_line + 1 to max(i2 + 1) + - Make sure that the indentation is correct (e.g. add DEDENT) + - Add old and change positions + Insert: + - Parse from parsed_until_line + 1 to min(j2 + 1), hopefully not + much more. + + Returns the new module node. + ''' + LOG.debug('diff parser start') + # Reset the used names cache so they get regenerated. + self._module._used_names = None + + self._parser_lines_new = new_lines + + self._reset() + + line_length = len(new_lines) + sm = difflib.SequenceMatcher(None, old_lines, self._parser_lines_new) + opcodes = sm.get_opcodes() + LOG.debug('diff parser calculated') + LOG.debug('diff: line_lengths old: %s, new: %s' % (len(old_lines), line_length)) + + for operation, i1, i2, j1, j2 in opcodes: + LOG.debug('diff %s old[%s:%s] new[%s:%s]', + operation, i1 + 1, i2, j1 + 1, j2) + + if j2 == line_length and new_lines[-1] == '': + # The empty part after the last newline is not relevant. + j2 -= 1 + + if operation == 'equal': + line_offset = j1 - i1 + self._copy_from_old_parser(line_offset, i2, j2) + elif operation == 'replace': + self._parse(until_line=j2) + elif operation == 'insert': + self._parse(until_line=j2) + else: + assert operation == 'delete' + + # With this action all change will finally be applied and we have a + # changed module. + self._nodes_stack.close() + + last_pos = self._module.end_pos[0] + if last_pos != line_length: + current_lines = split_lines(self._module.get_code(), keepends=True) + diff = difflib.unified_diff(current_lines, new_lines) + raise Exception( + "There's an issue (%s != %s) with the diff parser. Please report:\n%s" + % (last_pos, line_length, ''.join(diff)) + ) + + LOG.debug('diff parser end') + return self._module + + def _enabled_debugging(self, old_lines, lines_new): + if self._module.get_code() != ''.join(lines_new): + LOG.warning('parser issue:\n%s\n%s', ''.join(old_lines), + ''.join(lines_new)) + + def _copy_from_old_parser(self, line_offset, until_line_old, until_line_new): + copied_nodes = [None] + + last_until_line = -1 + while until_line_new > self._nodes_stack.parsed_until_line: + parsed_until_line_old = self._nodes_stack.parsed_until_line - line_offset + line_stmt = self._get_old_line_stmt(parsed_until_line_old + 1) + if line_stmt is None: + # Parse 1 line at least. We don't need more, because we just + # want to get into a state where the old parser has statements + # again that can be copied (e.g. not lines within parentheses). + self._parse(self._nodes_stack.parsed_until_line + 1) + elif not copied_nodes: + # We have copied as much as possible (but definitely not too + # much). Therefore we just parse the rest. + # We might not reach the end, because there's a statement + # that is not finished. + self._parse(until_line_new) + else: + p_children = line_stmt.parent.children + index = p_children.index(line_stmt) + + copied_nodes = self._nodes_stack.copy_nodes( + p_children[index:], + until_line_old, + line_offset + ) + # Match all the nodes that are in the wanted range. + if copied_nodes: + self._copy_count += 1 + + from_ = copied_nodes[0].get_start_pos_of_prefix()[0] + line_offset + to = self._nodes_stack.parsed_until_line + + LOG.debug('diff actually copy %s to %s', from_, to) + # Since there are potential bugs that might loop here endlessly, we + # just stop here. + assert last_until_line != self._nodes_stack.parsed_until_line \ + or not copied_nodes, last_until_line + last_until_line = self._nodes_stack.parsed_until_line + + def _get_old_line_stmt(self, old_line): + leaf = self._module.get_leaf_for_position((old_line, 0), include_prefixes=True) + + if _ends_with_newline(leaf): + leaf = leaf.get_next_leaf() + if leaf.get_start_pos_of_prefix()[0] == old_line: + node = leaf + while node.parent.type not in ('file_input', 'suite'): + node = node.parent + return node + # Must be on the same line. Otherwise we need to parse that bit. + return None + + def _get_before_insertion_node(self): + if self._nodes_stack.is_empty(): + return None + + line = self._nodes_stack.parsed_until_line + 1 + node = self._new_module.get_last_leaf() + while True: + parent = node.parent + if parent.type in ('suite', 'file_input'): + assert node.end_pos[0] <= line + assert node.end_pos[1] == 0 or '\n' in self._prefix + return node + node = parent + + def _parse(self, until_line): + """ + Parses at least until the given line, but might just parse more until a + valid state is reached. + """ + last_until_line = 0 + while until_line > self._nodes_stack.parsed_until_line: + node = self._try_parse_part(until_line) + nodes = node.children + + self._nodes_stack.add_parsed_nodes(nodes) + LOG.debug( + 'parse_part from %s to %s (to %s in part parser)', + nodes[0].get_start_pos_of_prefix()[0], + self._nodes_stack.parsed_until_line, + node.end_pos[0] - 1 + ) + # Since the tokenizer sometimes has bugs, we cannot be sure that + # this loop terminates. Therefore assert that there's always a + # change. + assert last_until_line != self._nodes_stack.parsed_until_line, last_until_line + last_until_line = self._nodes_stack.parsed_until_line + + def _try_parse_part(self, until_line): + """ + Sets up a normal parser that uses a spezialized tokenizer to only parse + until a certain position (or a bit longer if the statement hasn't + ended. + """ + self._parser_count += 1 + # TODO speed up, shouldn't copy the whole list all the time. + # memoryview? + parsed_until_line = self._nodes_stack.parsed_until_line + lines_after = self._parser_lines_new[parsed_until_line:] + #print('parse_content', parsed_until_line, lines_after, until_line) + tokens = self._diff_tokenize( + lines_after, + until_line, + line_offset=parsed_until_line + ) + self._active_parser = Parser( + self._pgen_grammar, + error_recovery=True + ) + return self._active_parser.parse(tokens=tokens) + + def _diff_tokenize(self, lines, until_line, line_offset=0): + is_first_token = True + omitted_first_indent = False + indents = [] + tokens = self._tokenizer(lines, (1, 0)) + stack = self._active_parser.pgen_parser.stack + for typ, string, start_pos, prefix in tokens: + start_pos = start_pos[0] + line_offset, start_pos[1] + if typ == INDENT: + indents.append(start_pos[1]) + if is_first_token: + omitted_first_indent = True + # We want to get rid of indents that are only here because + # we only parse part of the file. These indents would only + # get parsed as error leafs, which doesn't make any sense. + is_first_token = False + continue + is_first_token = False + + # In case of omitted_first_indent, it might not be dedented fully. + # However this is a sign for us that a dedent happened. + if typ == DEDENT \ + or typ == ERROR_DEDENT and omitted_first_indent and len(indents) == 1: + indents.pop() + if omitted_first_indent and not indents: + # We are done here, only thing that can come now is an + # endmarker or another dedented code block. + typ, string, start_pos, prefix = next(tokens) + if '\n' in prefix: + prefix = re.sub(r'(<=\n)[^\n]+$', '', prefix) + else: + prefix = '' + yield PythonToken(ENDMARKER, '', (start_pos[0] + line_offset, 0), prefix) + break + elif typ == NEWLINE and start_pos[0] >= until_line: + yield PythonToken(typ, string, start_pos, prefix) + # Check if the parser is actually in a valid suite state. + if suite_or_file_input_is_valid(self._pgen_grammar, stack): + start_pos = start_pos[0] + 1, 0 + while len(indents) > int(omitted_first_indent): + indents.pop() + yield PythonToken(DEDENT, '', start_pos, '') + + yield PythonToken(ENDMARKER, '', start_pos, '') + break + else: + continue + + yield PythonToken(typ, string, start_pos, prefix) + + +class _NodesStackNode(object): + ChildrenGroup = namedtuple('ChildrenGroup', 'children line_offset last_line_offset_leaf') + + def __init__(self, tree_node, parent=None): + self.tree_node = tree_node + self.children_groups = [] + self.parent = parent + + def close(self): + children = [] + for children_part, line_offset, last_line_offset_leaf in self.children_groups: + if line_offset != 0: + try: + _update_positions( + children_part, line_offset, last_line_offset_leaf) + except _PositionUpdatingFinished: + pass + children += children_part + self.tree_node.children = children + # Reset the parents + for node in children: + node.parent = self.tree_node + + def add(self, children, line_offset=0, last_line_offset_leaf=None): + group = self.ChildrenGroup(children, line_offset, last_line_offset_leaf) + self.children_groups.append(group) + + def get_last_line(self, suffix): + line = 0 + if self.children_groups: + children_group = self.children_groups[-1] + last_leaf = children_group.children[-1].get_last_leaf() + line = last_leaf.end_pos[0] + + # Calculate the line offsets + offset = children_group.line_offset + if offset: + # In case the line_offset is not applied to this specific leaf, + # just ignore it. + if last_leaf.line <= children_group.last_line_offset_leaf.line: + line += children_group.line_offset + + # Newlines end on the next line, which means that they would cover + # the next line. That line is not fully parsed at this point. + if _ends_with_newline(last_leaf, suffix): + line -= 1 + line += suffix.count('\n') + if suffix and not suffix.endswith('\n'): + # This is the end of a file (that doesn't end with a newline). + line += 1 + return line + + +class _NodesStack(object): + endmarker_type = 'endmarker' + + def __init__(self, module): + # Top of stack + self._tos = self._base_node = _NodesStackNode(module) + self._module = module + self._last_prefix = '' + self.prefix = '' + + def is_empty(self): + return not self._base_node.children + + @property + def parsed_until_line(self): + return self._tos.get_last_line(self.prefix) + + def _get_insertion_node(self, indentation_node): + indentation = indentation_node.start_pos[1] + + # find insertion node + node = self._tos + while True: + tree_node = node.tree_node + if tree_node.type == 'suite': + # A suite starts with NEWLINE, ... + node_indentation = tree_node.children[1].start_pos[1] + + if indentation >= node_indentation: # Not a Dedent + # We might be at the most outer layer: modules. We + # don't want to depend on the first statement + # having the right indentation. + return node + + elif tree_node.type == 'file_input': + return node + + node = self._close_tos() + + def _close_tos(self): + self._tos.close() + self._tos = self._tos.parent + return self._tos + + def add_parsed_nodes(self, tree_nodes): + tree_nodes = self._remove_endmarker(tree_nodes) + if not tree_nodes: + return + + assert tree_nodes[0].type != 'newline' + + node = self._get_insertion_node(tree_nodes[0]) + assert node.tree_node.type in ('suite', 'file_input') + node.add(tree_nodes) + self._update_tos(tree_nodes[-1]) + + def _remove_endmarker(self, tree_nodes): + """ + Helps cleaning up the tree nodes that get inserted. + """ + last_leaf = tree_nodes[-1].get_last_leaf() + is_endmarker = last_leaf.type == self.endmarker_type + self._last_prefix = '' + if is_endmarker: + try: + separation = last_leaf.prefix.rindex('\n') + except ValueError: + pass + else: + # Remove the whitespace part of the prefix after a newline. + # That is not relevant if parentheses were opened. Always parse + # until the end of a line. + last_leaf.prefix, self._last_prefix = \ + last_leaf.prefix[:separation + 1], last_leaf.prefix[separation + 1:] + + first_leaf = tree_nodes[0].get_first_leaf() + first_leaf.prefix = self.prefix + first_leaf.prefix + self.prefix = '' + + if is_endmarker: + self.prefix = last_leaf.prefix + + tree_nodes = tree_nodes[:-1] + + return tree_nodes + + def copy_nodes(self, tree_nodes, until_line, line_offset): + """ + Copies tree nodes from the old parser tree. + + Returns the number of tree nodes that were copied. + """ + tos = self._get_insertion_node(tree_nodes[0]) + + new_nodes, self._tos = self._copy_nodes(tos, tree_nodes, until_line, line_offset) + return new_nodes + + def _copy_nodes(self, tos, nodes, until_line, line_offset): + new_nodes = [] + + new_tos = tos + for node in nodes: + if node.type == 'endmarker': + # Endmarkers just distort all the checks below. Remove them. + break + + if node.start_pos[0] > until_line: + break + # TODO this check might take a bit of time for large files. We + # might want to change this to do more intelligent guessing or + # binary search. + if _get_last_line(node) > until_line: + # We can split up functions and classes later. + if node.type in ('classdef', 'funcdef') and node.children[-1].type == 'suite': + new_nodes.append(node) + break + + new_nodes.append(node) + + if not new_nodes: + return [], tos + + last_node = new_nodes[-1] + line_offset_index = -1 + if last_node.type in ('classdef', 'funcdef'): + suite = last_node.children[-1] + if suite.type == 'suite': + suite_tos = _NodesStackNode(suite) + # Don't need to pass line_offset here, it's already done by the + # parent. + suite_nodes, recursive_tos = self._copy_nodes( + suite_tos, suite.children, until_line, line_offset) + if len(suite_nodes) < 2: + # A suite only with newline is not valid. + new_nodes.pop() + else: + suite_tos.parent = tos + new_tos = recursive_tos + line_offset_index = -2 + + elif (new_nodes[-1].type in ('error_leaf', 'error_node') or + _is_flow_node(new_nodes[-1])): + # Error leafs/nodes don't have a defined start/end. Error + # nodes might not end with a newline (e.g. if there's an + # open `(`). Therefore ignore all of them unless they are + # succeeded with valid parser state. + # If we copy flows at the end, they might be continued + # after the copy limit (in the new parser). + # In this while loop we try to remove until we find a newline. + new_nodes.pop() + while new_nodes: + last_node = new_nodes[-1] + if last_node.get_last_leaf().type == 'newline': + break + new_nodes.pop() + + if new_nodes: + try: + last_line_offset_leaf = new_nodes[line_offset_index].get_last_leaf() + except IndexError: + line_offset = 0 + # In this case we don't have to calculate an offset, because + # there's no children to be managed. + last_line_offset_leaf = None + tos.add(new_nodes, line_offset, last_line_offset_leaf) + return new_nodes, new_tos + + def _update_tos(self, tree_node): + if tree_node.type in ('suite', 'file_input'): + self._tos = _NodesStackNode(tree_node, self._tos) + self._tos.add(list(tree_node.children)) + self._update_tos(tree_node.children[-1]) + elif tree_node.type in ('classdef', 'funcdef'): + self._update_tos(tree_node.children[-1]) + + def close(self): + while self._tos is not None: + self._close_tos() + + # Add an endmarker. + try: + last_leaf = self._module.get_last_leaf() + end_pos = list(last_leaf.end_pos) + except IndexError: + end_pos = [1, 0] + lines = split_lines(self.prefix) + assert len(lines) > 0 + if len(lines) == 1: + end_pos[1] += len(lines[0]) + else: + end_pos[0] += len(lines) - 1 + end_pos[1] = len(lines[-1]) + + endmarker = EndMarker('', tuple(end_pos), self.prefix + self._last_prefix) + endmarker.parent = self._module + self._module.children.append(endmarker) diff --git a/pythonFiles/parso/python/errors.py b/pythonFiles/parso/python/errors.py new file mode 100644 index 000000000000..65296568b54c --- /dev/null +++ b/pythonFiles/parso/python/errors.py @@ -0,0 +1,1053 @@ +# -*- coding: utf-8 -*- +import codecs +import warnings +import re +from contextlib import contextmanager + +from parso.normalizer import Normalizer, NormalizerConfig, Issue, Rule +from parso.python.tree import search_ancestor +from parso.parser import ParserSyntaxError + +_BLOCK_STMTS = ('if_stmt', 'while_stmt', 'for_stmt', 'try_stmt', 'with_stmt') +_STAR_EXPR_PARENTS = ('testlist_star_expr', 'testlist_comp', 'exprlist') +# This is the maximal block size given by python. +_MAX_BLOCK_SIZE = 20 +_MAX_INDENT_COUNT = 100 +ALLOWED_FUTURES = ( + 'all_feature_names', 'nested_scopes', 'generators', 'division', + 'absolute_import', 'with_statement', 'print_function', 'unicode_literals', +) + + +def _iter_stmts(scope): + """ + Iterates over all statements and splits up simple_stmt. + """ + for child in scope.children: + if child.type == 'simple_stmt': + for child2 in child.children: + if child2.type == 'newline' or child2 == ';': + continue + yield child2 + else: + yield child + + +def _get_comprehension_type(atom): + first, second = atom.children[:2] + if second.type == 'testlist_comp' and second.children[1].type == 'comp_for': + if first == '[': + return 'list comprehension' + else: + return 'generator expression' + elif second.type == 'dictorsetmaker' and second.children[-1].type == 'comp_for': + if second.children[1] == ':': + return 'dict comprehension' + else: + return 'set comprehension' + return None + + +def _is_future_import(import_from): + # It looks like a __future__ import that is relative is still a future + # import. That feels kind of odd, but whatever. + # if import_from.level != 0: + # return False + from_names = import_from.get_from_names() + return [n.value for n in from_names] == ['__future__'] + + +def _remove_parens(atom): + """ + Returns the inner part of an expression like `(foo)`. Also removes nested + parens. + """ + try: + children = atom.children + except AttributeError: + pass + else: + if len(children) == 3 and children[0] == '(': + return _remove_parens(atom.children[1]) + return atom + + +def _iter_params(parent_node): + return (n for n in parent_node.children if n.type == 'param') + + +def _is_future_import_first(import_from): + """ + Checks if the import is the first statement of a file. + """ + found_docstring = False + for stmt in _iter_stmts(import_from.get_root_node()): + if stmt.type == 'string' and not found_docstring: + continue + found_docstring = True + + if stmt == import_from: + return True + if stmt.type == 'import_from' and _is_future_import(stmt): + continue + return False + + +def _iter_definition_exprs_from_lists(exprlist): + for child in exprlist.children[::2]: + if child.type == 'atom' and child.children[0] in ('(', '['): + testlist_comp = child.children[0] + if testlist_comp.type == 'testlist_comp': + for expr in _iter_definition_exprs_from_lists(testlist_comp): + yield expr + continue + elif child.children[0] == '[': + yield testlist_comp + continue + + yield child + +def _get_expr_stmt_definition_exprs(expr_stmt): + exprs = [] + for list_ in expr_stmt.children[:-2:2]: + if list_.type in ('testlist_star_expr', 'testlist'): + exprs += _iter_definition_exprs_from_lists(list_) + else: + exprs.append(list_) + return exprs + + +def _get_for_stmt_definition_exprs(for_stmt): + exprlist = for_stmt.children[1] + if exprlist.type != 'exprlist': + return [exprlist] + return list(_iter_definition_exprs_from_lists(exprlist)) + + +class _Context(object): + def __init__(self, node, add_syntax_error, parent_context=None): + self.node = node + self.blocks = [] + self.parent_context = parent_context + self._used_name_dict = {} + self._global_names = [] + self._nonlocal_names = [] + self._nonlocal_names_in_subscopes = [] + self._add_syntax_error = add_syntax_error + + def is_async_funcdef(self): + # Stupidly enough async funcdefs can have two different forms, + # depending if a decorator is used or not. + return self.is_function() \ + and self.node.parent.type in ('async_funcdef', 'async_stmt') + + def is_function(self): + return self.node.type == 'funcdef' + + def add_name(self, name): + parent_type = name.parent.type + if parent_type == 'trailer': + # We are only interested in first level names. + return + + if parent_type == 'global_stmt': + self._global_names.append(name) + elif parent_type == 'nonlocal_stmt': + self._nonlocal_names.append(name) + else: + self._used_name_dict.setdefault(name.value, []).append(name) + + def finalize(self): + """ + Returns a list of nonlocal names that need to be part of that scope. + """ + self._analyze_names(self._global_names, 'global') + self._analyze_names(self._nonlocal_names, 'nonlocal') + + # Python2.6 doesn't have dict comprehensions. + global_name_strs = dict((n.value, n) for n in self._global_names) + for nonlocal_name in self._nonlocal_names: + try: + global_name = global_name_strs[nonlocal_name.value] + except KeyError: + continue + + message = "name '%s' is nonlocal and global" % global_name.value + if global_name.start_pos < nonlocal_name.start_pos: + error_name = global_name + else: + error_name = nonlocal_name + self._add_syntax_error(error_name, message) + + nonlocals_not_handled = [] + for nonlocal_name in self._nonlocal_names_in_subscopes: + search = nonlocal_name.value + if search in global_name_strs or self.parent_context is None: + message = "no binding for nonlocal '%s' found" % nonlocal_name.value + self._add_syntax_error(nonlocal_name, message) + elif not self.is_function() or \ + nonlocal_name.value not in self._used_name_dict: + nonlocals_not_handled.append(nonlocal_name) + return self._nonlocal_names + nonlocals_not_handled + + def _analyze_names(self, globals_or_nonlocals, type_): + def raise_(message): + self._add_syntax_error(base_name, message % (base_name.value, type_)) + + params = [] + if self.node.type == 'funcdef': + params = self.node.get_params() + + for base_name in globals_or_nonlocals: + found_global_or_nonlocal = False + # Somehow Python does it the reversed way. + for name in reversed(self._used_name_dict.get(base_name.value, [])): + if name.start_pos > base_name.start_pos: + # All following names don't have to be checked. + found_global_or_nonlocal = True + + parent = name.parent + if parent.type == 'param' and parent.name == name: + # Skip those here, these definitions belong to the next + # scope. + continue + + if name.is_definition(): + if parent.type == 'expr_stmt' \ + and parent.children[1].type == 'annassign': + if found_global_or_nonlocal: + # If it's after the global the error seems to be + # placed there. + base_name = name + raise_("annotated name '%s' can't be %s") + break + else: + message = "name '%s' is assigned to before %s declaration" + else: + message = "name '%s' is used prior to %s declaration" + + if not found_global_or_nonlocal: + raise_(message) + # Only add an error for the first occurence. + break + + for param in params: + if param.name.value == base_name.value: + raise_("name '%s' is parameter and %s"), + + @contextmanager + def add_block(self, node): + self.blocks.append(node) + yield + self.blocks.pop() + + def add_context(self, node): + return _Context(node, self._add_syntax_error, parent_context=self) + + def close_child_context(self, child_context): + self._nonlocal_names_in_subscopes += child_context.finalize() + + +class ErrorFinder(Normalizer): + """ + Searches for errors in the syntax tree. + """ + def __init__(self, *args, **kwargs): + super(ErrorFinder, self).__init__(*args, **kwargs) + self._error_dict = {} + self.version = self.grammar.version_info + + def initialize(self, node): + def create_context(node): + if node is None: + return None + + parent_context = create_context(node.parent) + if node.type in ('classdef', 'funcdef', 'file_input'): + return _Context(node, self._add_syntax_error, parent_context) + return parent_context + + self.context = create_context(node) or _Context(node, self._add_syntax_error) + self._indentation_count = 0 + + def visit(self, node): + if node.type == 'error_node': + with self.visit_node(node): + # Don't need to investigate the inners of an error node. We + # might find errors in there that should be ignored, because + # the error node itself already shows that there's an issue. + return '' + return super(ErrorFinder, self).visit(node) + + + @contextmanager + def visit_node(self, node): + self._check_type_rules(node) + + if node.type in _BLOCK_STMTS: + with self.context.add_block(node): + if len(self.context.blocks) == _MAX_BLOCK_SIZE: + self._add_syntax_error(node, "too many statically nested blocks") + yield + return + elif node.type == 'suite': + self._indentation_count += 1 + if self._indentation_count == _MAX_INDENT_COUNT: + self._add_indentation_error(node.children[1], "too many levels of indentation") + + yield + + if node.type == 'suite': + self._indentation_count -= 1 + elif node.type in ('classdef', 'funcdef'): + context = self.context + self.context = context.parent_context + self.context.close_child_context(context) + + def visit_leaf(self, leaf): + if leaf.type == 'error_leaf': + if leaf.original_type in ('indent', 'error_dedent'): + # Indents/Dedents itself never have a prefix. They are just + # "pseudo" tokens that get removed by the syntax tree later. + # Therefore in case of an error we also have to check for this. + spacing = list(leaf.get_next_leaf()._split_prefix())[-1] + if leaf.original_type == 'indent': + message = 'unexpected indent' + else: + message = 'unindent does not match any outer indentation level' + self._add_indentation_error(spacing, message) + else: + if leaf.value.startswith('\\'): + message = 'unexpected character after line continuation character' + else: + match = re.match('\\w{,2}("{1,3}|\'{1,3})', leaf.value) + if match is None: + message = 'invalid syntax' + else: + if len(match.group(1)) == 1: + message = 'EOL while scanning string literal' + else: + message = 'EOF while scanning triple-quoted string literal' + self._add_syntax_error(leaf, message) + return '' + elif leaf.value == ':': + parent = leaf.parent + if parent.type in ('classdef', 'funcdef'): + self.context = self.context.add_context(parent) + + # The rest is rule based. + return super(ErrorFinder, self).visit_leaf(leaf) + + def _add_indentation_error(self, spacing, message): + self.add_issue(spacing, 903, "IndentationError: " + message) + + def _add_syntax_error(self, node, message): + self.add_issue(node, 901, "SyntaxError: " + message) + + def add_issue(self, node, code, message): + # Overwrite the default behavior. + # Check if the issues are on the same line. + line = node.start_pos[0] + args = (code, message, node) + self._error_dict.setdefault(line, args) + + def finalize(self): + self.context.finalize() + + for code, message, node in self._error_dict.values(): + self.issues.append(Issue(node, code, message)) + + +class IndentationRule(Rule): + code = 903 + + def _get_message(self, message): + message = super(IndentationRule, self)._get_message(message) + return "IndentationError: " + message + + +@ErrorFinder.register_rule(type='error_node') +class _ExpectIndentedBlock(IndentationRule): + message = 'expected an indented block' + + def get_node(self, node): + leaf = node.get_next_leaf() + return list(leaf._split_prefix())[-1] + + def is_issue(self, node): + # This is the beginning of a suite that is not indented. + return node.children[-1].type == 'newline' + + +class ErrorFinderConfig(NormalizerConfig): + normalizer_class = ErrorFinder + + +class SyntaxRule(Rule): + code = 901 + + def _get_message(self, message): + message = super(SyntaxRule, self)._get_message(message) + return "SyntaxError: " + message + + +@ErrorFinder.register_rule(type='error_node') +class _InvalidSyntaxRule(SyntaxRule): + message = "invalid syntax" + + def get_node(self, node): + return node.get_next_leaf() + + def is_issue(self, node): + # Error leafs will be added later as an error. + return node.get_next_leaf().type != 'error_leaf' + + +@ErrorFinder.register_rule(value='await') +class _AwaitOutsideAsync(SyntaxRule): + message = "'await' outside async function" + + def is_issue(self, leaf): + return not self._normalizer.context.is_async_funcdef() + + def get_error_node(self, node): + # Return the whole await statement. + return node.parent + + +@ErrorFinder.register_rule(value='break') +class _BreakOutsideLoop(SyntaxRule): + message = "'break' outside loop" + + def is_issue(self, leaf): + in_loop = False + for block in self._normalizer.context.blocks: + if block.type in ('for_stmt', 'while_stmt'): + in_loop = True + return not in_loop + + +@ErrorFinder.register_rule(value='continue') +class _ContinueChecks(SyntaxRule): + message = "'continue' not properly in loop" + message_in_finally = "'continue' not supported inside 'finally' clause" + + def is_issue(self, leaf): + in_loop = False + for block in self._normalizer.context.blocks: + if block.type in ('for_stmt', 'while_stmt'): + in_loop = True + if block.type == 'try_stmt': + last_block = block.children[-3] + if last_block == 'finally' and leaf.start_pos > last_block.start_pos: + self.add_issue(leaf, message=self.message_in_finally) + return False # Error already added + if not in_loop: + return True + + +@ErrorFinder.register_rule(value='from') +class _YieldFromCheck(SyntaxRule): + message = "'yield from' inside async function" + + def get_node(self, leaf): + return leaf.parent.parent # This is the actual yield statement. + + def is_issue(self, leaf): + return leaf.parent.type == 'yield_arg' \ + and self._normalizer.context.is_async_funcdef() + + +@ErrorFinder.register_rule(type='name') +class _NameChecks(SyntaxRule): + message = 'cannot assign to __debug__' + message_keyword = 'assignment to keyword' + message_none = 'cannot assign to None' + + def is_issue(self, leaf): + self._normalizer.context.add_name(leaf) + + if leaf.value == '__debug__' and leaf.is_definition(): + if self._normalizer.version < (3, 0): + return True + else: + self.add_issue(leaf, message=self.message_keyword) + if leaf.value == 'None' and self._normalizer.version < (3, 0) \ + and leaf.is_definition(): + self.add_issue(leaf, message=self.message_none) + + +@ErrorFinder.register_rule(type='string') +class _StringChecks(SyntaxRule): + message = "bytes can only contain ASCII literal characters." + + def is_issue(self, leaf): + string_prefix = leaf.string_prefix.lower() + if 'b' in string_prefix \ + and self._normalizer.version >= (3, 0) \ + and any(c for c in leaf.value if ord(c) > 127): + # b'ä' + return True + + if 'r' not in string_prefix: + # Raw strings don't need to be checked if they have proper + # escaping. + is_bytes = self._normalizer.version < (3, 0) + if 'b' in string_prefix: + is_bytes = True + if 'u' in string_prefix: + is_bytes = False + + payload = leaf._get_payload() + if is_bytes: + payload = payload.encode('utf-8') + func = codecs.escape_decode + else: + func = codecs.unicode_escape_decode + + try: + with warnings.catch_warnings(): + # The warnings from parsing strings are not relevant. + warnings.filterwarnings('ignore') + func(payload) + except UnicodeDecodeError as e: + self.add_issue(leaf, message='(unicode error) ' + str(e)) + except ValueError as e: + self.add_issue(leaf, message='(value error) ' + str(e)) + + +@ErrorFinder.register_rule(value='*') +class _StarCheck(SyntaxRule): + message = "named arguments must follow bare *" + + def is_issue(self, leaf): + params = leaf.parent + if params.type == 'parameters' and params: + after = params.children[params.children.index(leaf) + 1:] + after = [child for child in after + if child not in (',', ')') and not child.star_count] + return len(after) == 0 + + +@ErrorFinder.register_rule(value='**') +class _StarStarCheck(SyntaxRule): + # e.g. {**{} for a in [1]} + # TODO this should probably get a better end_pos including + # the next sibling of leaf. + message = "dict unpacking cannot be used in dict comprehension" + + def is_issue(self, leaf): + if leaf.parent.type == 'dictorsetmaker': + comp_for = leaf.get_next_sibling().get_next_sibling() + return comp_for is not None and comp_for.type == 'comp_for' + + +@ErrorFinder.register_rule(value='yield') +@ErrorFinder.register_rule(value='return') +class _ReturnAndYieldChecks(SyntaxRule): + message = "'return' with value in async generator" + message_async_yield = "'yield' inside async function" + + def get_node(self, leaf): + return leaf.parent + + def is_issue(self, leaf): + if self._normalizer.context.node.type != 'funcdef': + self.add_issue(self.get_node(leaf), message="'%s' outside function" % leaf.value) + elif self._normalizer.context.is_async_funcdef() \ + and any(self._normalizer.context.node.iter_yield_exprs()): + if leaf.value == 'return' and leaf.parent.type == 'return_stmt': + return True + elif leaf.value == 'yield' \ + and leaf.get_next_leaf() != 'from' \ + and self._normalizer.version == (3, 5): + self.add_issue(self.get_node(leaf), message=self.message_async_yield) + +@ErrorFinder.register_rule(type='atom') +class _BytesAndStringMix(SyntaxRule): + # e.g. 's' b'' + message = "cannot mix bytes and nonbytes literals" + + def _is_bytes_literal(self, string): + return 'b' in string.string_prefix.lower() + + def is_issue(self, node): + first = node.children[0] + if first.type == 'string' and self._normalizer.version >= (3, 0): + first_is_bytes = self._is_bytes_literal(first) + for string in node.children[1:]: + if first_is_bytes != self._is_bytes_literal(string): + return True + + +@ErrorFinder.register_rule(type='import_as_names') +class _TrailingImportComma(SyntaxRule): + # e.g. from foo import a, + message = "trailing comma not allowed without surrounding parentheses" + + def is_issue(self, node): + if node.children[-1] == ',': + return True + + +@ErrorFinder.register_rule(type='import_from') +class _ImportStarInFunction(SyntaxRule): + message = "import * only allowed at module level" + + def is_issue(self, node): + return node.is_star_import() and self._normalizer.context.parent_context is not None + + +@ErrorFinder.register_rule(type='import_from') +class _FutureImportRule(SyntaxRule): + message = "from __future__ imports must occur at the beginning of the file" + + def is_issue(self, node): + if _is_future_import(node): + if not _is_future_import_first(node): + return True + + for from_name, future_name in node.get_paths(): + name = future_name.value + allowed_futures = list(ALLOWED_FUTURES) + if self._normalizer.version >= (3, 5): + allowed_futures.append('generator_stop') + + if name == 'braces': + self.add_issue(node, message = "not a chance") + elif name == 'barry_as_FLUFL': + m = "Seriously I'm not implementing this :) ~ Dave" + self.add_issue(node, message=m) + elif name not in ALLOWED_FUTURES: + message = "future feature %s is not defined" % name + self.add_issue(node, message=message) + + +@ErrorFinder.register_rule(type='star_expr') +class _StarExprRule(SyntaxRule): + message = "starred assignment target must be in a list or tuple" + message_iterable_unpacking = "iterable unpacking cannot be used in comprehension" + message_assignment = "can use starred expression only as assignment target" + + def is_issue(self, node): + if node.parent.type not in _STAR_EXPR_PARENTS: + return True + if node.parent.type == 'testlist_comp': + # [*[] for a in [1]] + if node.parent.children[1].type == 'comp_for': + self.add_issue(node, message=self.message_iterable_unpacking) + if self._normalizer.version <= (3, 4): + n = search_ancestor(node, 'for_stmt', 'expr_stmt') + found_definition = False + if n is not None: + if n.type == 'expr_stmt': + exprs = _get_expr_stmt_definition_exprs(n) + else: + exprs = _get_for_stmt_definition_exprs(n) + if node in exprs: + found_definition = True + + if not found_definition: + self.add_issue(node, message=self.message_assignment) + + +@ErrorFinder.register_rule(types=_STAR_EXPR_PARENTS) +class _StarExprParentRule(SyntaxRule): + def is_issue(self, node): + if node.parent.type == 'del_stmt': + self.add_issue(node.parent, message="can't use starred expression here") + else: + def is_definition(node, ancestor): + if ancestor is None: + return False + + type_ = ancestor.type + if type_ == 'trailer': + return False + + if type_ == 'expr_stmt': + return node.start_pos < ancestor.children[-1].start_pos + + return is_definition(node, ancestor.parent) + + if is_definition(node, node.parent): + args = [c for c in node.children if c != ','] + starred = [c for c in args if c.type == 'star_expr'] + if len(starred) > 1: + message = "two starred expressions in assignment" + self.add_issue(starred[1], message=message) + elif starred: + count = args.index(starred[0]) + if count >= 256: + message = "too many expressions in star-unpacking assignment" + self.add_issue(starred[0], message=message) + + +@ErrorFinder.register_rule(type='annassign') +class _AnnotatorRule(SyntaxRule): + # True: int + # {}: float + message = "illegal target for annotation" + + def get_node(self, node): + return node.parent + + def is_issue(self, node): + type_ = None + lhs = node.parent.children[0] + lhs = _remove_parens(lhs) + try: + children = lhs.children + except AttributeError: + pass + else: + if ',' in children or lhs.type == 'atom' and children[0] == '(': + type_ = 'tuple' + elif lhs.type == 'atom' and children[0] == '[': + type_ = 'list' + trailer = children[-1] + + if type_ is None: + if not (lhs.type == 'name' + # subscript/attributes are allowed + or lhs.type in ('atom_expr', 'power') + and trailer.type == 'trailer' + and trailer.children[0] != '('): + return True + else: + # x, y: str + message = "only single target (not %s) can be annotated" + self.add_issue(lhs.parent, message=message % type_) + + +@ErrorFinder.register_rule(type='argument') +class _ArgumentRule(SyntaxRule): + def is_issue(self, node): + first = node.children[0] + if node.children[1] == '=' and first.type != 'name': + if first.type == 'lambdef': + # f(lambda: 1=1) + message = "lambda cannot contain assignment" + else: + # f(+x=1) + message = "keyword can't be an expression" + self.add_issue(first, message=message) + + +@ErrorFinder.register_rule(type='nonlocal_stmt') +class _NonlocalModuleLevelRule(SyntaxRule): + message = "nonlocal declaration not allowed at module level" + + def is_issue(self, node): + return self._normalizer.context.parent_context is None + + +@ErrorFinder.register_rule(type='arglist') +class _ArglistRule(SyntaxRule): + message = "Generator expression must be parenthesized if not sole argument" + + def is_issue(self, node): + first_arg = node.children[0] + if first_arg.type == 'argument' \ + and first_arg.children[1].type == 'comp_for': + # e.g. foo(x for x in [], b) + return len(node.children) >= 2 + else: + arg_set = set() + kw_only = False + kw_unpacking_only = False + is_old_starred = False + # In python 3 this would be a bit easier (stars are part of + # argument), but we have to understand both. + for argument in node.children: + if argument == ',': + continue + + if argument in ('*', '**'): + # Python < 3.5 has the order engraved in the grammar + # file. No need to do anything here. + is_old_starred = True + continue + if is_old_starred: + is_old_starred = False + continue + + if argument.type == 'argument': + first = argument.children[0] + if first in ('*', '**'): + if first == '*': + if kw_unpacking_only: + # foo(**kwargs, *args) + message = "iterable argument unpacking follows keyword argument unpacking" + self.add_issue(argument, message=message) + else: + kw_unpacking_only = True + else: # Is a keyword argument. + kw_only = True + if first.type == 'name': + if first.value in arg_set: + # f(x=1, x=2) + self.add_issue(first, message="keyword argument repeated") + else: + arg_set.add(first.value) + else: + if kw_unpacking_only: + # f(**x, y) + message = "positional argument follows keyword argument unpacking" + self.add_issue(argument, message=message) + elif kw_only: + # f(x=2, y) + message = "positional argument follows keyword argument" + self.add_issue(argument, message=message) + +@ErrorFinder.register_rule(type='parameters') +@ErrorFinder.register_rule(type='lambdef') +class _ParameterRule(SyntaxRule): + # def f(x=3, y): pass + message = "non-default argument follows default argument" + + def is_issue(self, node): + param_names = set() + default_only = False + for p in _iter_params(node): + if p.name.value in param_names: + message = "duplicate argument '%s' in function definition" + self.add_issue(p.name, message=message % p.name.value) + param_names.add(p.name.value) + + if p.default is None and not p.star_count: + if default_only: + return True + else: + default_only = True + + +@ErrorFinder.register_rule(type='try_stmt') +class _TryStmtRule(SyntaxRule): + message = "default 'except:' must be last" + + def is_issue(self, try_stmt): + default_except = None + for except_clause in try_stmt.children[3::3]: + if except_clause in ('else', 'finally'): + break + if except_clause == 'except': + default_except = except_clause + elif default_except is not None: + self.add_issue(default_except, message=self.message) + + +@ErrorFinder.register_rule(type='string') +class _FStringRule(SyntaxRule): + _fstring_grammar = None + message_empty = "f-string: empty expression not allowed" # f'{}' + message_single_closing = "f-string: single '}' is not allowed" # f'}' + message_nested = "f-string: expressions nested too deeply" + message_backslash = "f-string expression part cannot include a backslash" # f'{"\"}' or f'{"\\"}' + message_comment = "f-string expression part cannot include '#'" # f'{#}' + message_unterminated_string = "f-string: unterminated string" # f'{"}' + message_conversion = "f-string: invalid conversion character: expected 's', 'r', or 'a'" + message_incomplete = "f-string: expecting '}'" # f'{' + message_syntax = "invalid syntax" + + @classmethod + def _load_grammar(cls): + import parso + + if cls._fstring_grammar is None: + cls._fstring_grammar = parso.load_grammar(language='python-f-string') + return cls._fstring_grammar + + def is_issue(self, fstring): + if 'f' not in fstring.string_prefix.lower(): + return + + parsed = self._load_grammar().parse_leaf(fstring) + for child in parsed.children: + if child.type == 'expression': + self._check_expression(child) + elif child.type == 'error_node': + next_ = child.get_next_leaf() + if next_.type == 'error_leaf' and next_.original_type == 'unterminated_string': + self.add_issue(next_, message=self.message_unterminated_string) + # At this point nothing more is comming except the error + # leaf that we've already checked here. + break + self.add_issue(child, message=self.message_incomplete) + elif child.type == 'error_leaf': + self.add_issue(child, message=self.message_single_closing) + + def _check_python_expr(self, python_expr): + value = python_expr.value + if '\\' in value: + self.add_issue(python_expr, message=self.message_backslash) + return + if '#' in value: + self.add_issue(python_expr, message=self.message_comment) + return + if re.match('\s*$', value) is not None: + self.add_issue(python_expr, message=self.message_empty) + return + + # This is now nested parsing. We parsed the fstring and now + # we're parsing Python again. + try: + # CPython has a bit of a special ways to parse Python code within + # f-strings. It wraps the code in brackets to make sure that + # whitespace doesn't make problems (indentation/newlines). + # Just use that algorithm as well here and adapt start positions. + start_pos = python_expr.start_pos + start_pos = start_pos[0], start_pos[1] - 1 + eval_input = self._normalizer.grammar._parse( + '(%s)' % value, + start_symbol='eval_input', + start_pos=start_pos, + error_recovery=False + ) + except ParserSyntaxError as e: + self.add_issue(e.error_leaf, message=self.message_syntax) + return + + issues = self._normalizer.grammar.iter_errors(eval_input) + self._normalizer.issues += issues + + def _check_format_spec(self, format_spec): + for expression in format_spec.children[1:]: + nested_format_spec = expression.children[-2] + if nested_format_spec.type == 'format_spec': + if len(nested_format_spec.children) > 1: + self.add_issue( + nested_format_spec.children[1], + message=self.message_nested + ) + + self._check_expression(expression) + + def _check_expression(self, expression): + for c in expression.children: + if c.type == 'python_expr': + self._check_python_expr(c) + elif c.type == 'conversion': + if c.value not in ('s', 'r', 'a'): + self.add_issue(c, message=self.message_conversion) + elif c.type == 'format_spec': + self._check_format_spec(c) + + +class _CheckAssignmentRule(SyntaxRule): + def _check_assignment(self, node, is_deletion=False): + error = None + type_ = node.type + if type_ == 'lambdef': + error = 'lambda' + elif type_ == 'atom': + first, second = node.children[:2] + error = _get_comprehension_type(node) + if error is None: + if second.type in ('dictorsetmaker', 'string'): + error = 'literal' + elif first in ('(', '['): + if second.type == 'yield_expr': + error = 'yield expression' + elif second.type == 'testlist_comp': + # This is not a comprehension, they were handled + # further above. + for child in second.children[::2]: + self._check_assignment(child, is_deletion) + else: # Everything handled, must be useless brackets. + self._check_assignment(second, is_deletion) + elif type_ == 'keyword': + error = 'keyword' + elif type_ == 'operator': + if node.value == '...': + error = 'Ellipsis' + elif type_ == 'comparison': + error = 'comparison' + elif type_ in ('string', 'number'): + error = 'literal' + elif type_ == 'yield_expr': + # This one seems to be a slightly different warning in Python. + message = 'assignment to yield expression not possible' + self.add_issue(node, message=message) + elif type_ == 'test': + error = 'conditional expression' + elif type_ in ('atom_expr', 'power'): + if node.children[0] == 'await': + error = 'await expression' + elif node.children[-2] == '**': + error = 'operator' + else: + # Has a trailer + trailer = node.children[-1] + assert trailer.type == 'trailer' + if trailer.children[0] == '(': + error = 'function call' + elif type_ in ('testlist_star_expr', 'exprlist', 'testlist'): + for child in node.children[::2]: + self._check_assignment(child, is_deletion) + elif ('expr' in type_ and type_ != 'star_expr' # is a substring + or '_test' in type_ + or type_ in ('term', 'factor')): + error = 'operator' + + if error is not None: + message = "can't %s %s" % ("delete" if is_deletion else "assign to", error) + self.add_issue(node, message=message) + + +@ErrorFinder.register_rule(type='comp_for') +class _CompForRule(_CheckAssignmentRule): + message = "asynchronous comprehension outside of an asynchronous function" + + def is_issue(self, node): + # Some of the nodes here are already used, so no else if + expr_list = node.children[1 + int(node.children[0] == 'async')] + if expr_list.type != 'expr_list': # Already handled. + self._check_assignment(expr_list) + + return node.children[0] == 'async' \ + and not self._normalizer.context.is_async_funcdef() + + +@ErrorFinder.register_rule(type='expr_stmt') +class _ExprStmtRule(_CheckAssignmentRule): + message = "illegal expression for augmented assignment" + + def is_issue(self, node): + for before_equal in node.children[:-2:2]: + self._check_assignment(before_equal) + + augassign = node.children[1] + if augassign != '=' and augassign.type != 'annassign': # Is augassign. + return node.children[0].type in ('testlist_star_expr', 'atom', 'testlist') + + +@ErrorFinder.register_rule(type='with_item') +class _WithItemRule(_CheckAssignmentRule): + def is_issue(self, with_item): + self._check_assignment(with_item.children[2]) + + +@ErrorFinder.register_rule(type='del_stmt') +class _DelStmtRule(_CheckAssignmentRule): + def is_issue(self, del_stmt): + child = del_stmt.children[1] + + if child.type != 'expr_list': # Already handled. + self._check_assignment(child, is_deletion=True) + + +@ErrorFinder.register_rule(type='expr_list') +class _ExprListRule(_CheckAssignmentRule): + def is_issue(self, expr_list): + for expr in expr_list.children[::2]: + self._check_assignment(expr) + + +@ErrorFinder.register_rule(type='for_stmt') +class _ForStmtRule(_CheckAssignmentRule): + def is_issue(self, for_stmt): + # Some of the nodes here are already used, so no else if + expr_list = for_stmt.children[1] + if expr_list.type != 'expr_list': # Already handled. + self._check_assignment(expr_list) diff --git a/pythonFiles/parso/python/fstring.py b/pythonFiles/parso/python/fstring.py new file mode 100644 index 000000000000..a8fe7b452df5 --- /dev/null +++ b/pythonFiles/parso/python/fstring.py @@ -0,0 +1,211 @@ +import re + +from itertools import count +from parso.utils import PythonVersionInfo +from parso.utils import split_lines +from parso.python.tokenize import Token +from parso import parser +from parso.tree import TypedLeaf, ErrorNode, ErrorLeaf + +version36 = PythonVersionInfo(3, 6) + + +class TokenNamespace: + _c = count() + LBRACE = next(_c) + RBRACE = next(_c) + ENDMARKER = next(_c) + COLON = next(_c) + CONVERSION = next(_c) + PYTHON_EXPR = next(_c) + EXCLAMATION_MARK = next(_c) + UNTERMINATED_STRING = next(_c) + + token_map = dict((v, k) for k, v in locals().items() if not k.startswith('_')) + + @classmethod + def generate_token_id(cls, string): + if string == '{': + return cls.LBRACE + elif string == '}': + return cls.RBRACE + elif string == '!': + return cls.EXCLAMATION_MARK + elif string == ':': + return cls.COLON + return getattr(cls, string) + + +GRAMMAR = """ +fstring: expression* ENDMARKER +format_spec: ':' expression* +expression: '{' PYTHON_EXPR [ '!' CONVERSION ] [ format_spec ] '}' +""" + +_prefix = r'((?:[^{}]+)*)' +_expr = _prefix + r'(\{|\}|$)' +_in_expr = r'([^{}\[\]:"\'!]*)(.?)' +# There's only one conversion character allowed. But the rules have to be +# checked later anyway, so allow more here. This makes error recovery nicer. +_conversion = r'([^={}:]*)(.?)' + +_compiled_expr = re.compile(_expr) +_compiled_in_expr = re.compile(_in_expr) +_compiled_conversion = re.compile(_conversion) + + +def tokenize(code, start_pos=(1, 0)): + def add_to_pos(string): + lines = split_lines(string) + l = len(lines[-1]) + if len(lines) > 1: + start_pos[0] += len(lines) - 1 + start_pos[1] = l + else: + start_pos[1] += l + + def tok(value, type=None, prefix=''): + if type is None: + type = TokenNamespace.generate_token_id(value) + + add_to_pos(prefix) + token = Token(type, value, tuple(start_pos), prefix) + add_to_pos(value) + return token + + start = 0 + recursion_level = 0 + added_prefix = '' + start_pos = list(start_pos) + while True: + match = _compiled_expr.match(code, start) + prefix = added_prefix + match.group(1) + found = match.group(2) + start = match.end() + if not found: + # We're at the end. + break + + if found == '}': + if recursion_level == 0 and len(code) > start and code[start] == '}': + # This is a }} escape. + added_prefix = prefix + '}}' + start += 1 + continue + + recursion_level = max(0, recursion_level - 1) + yield tok(found, prefix=prefix) + added_prefix = '' + else: + assert found == '{' + if recursion_level == 0 and len(code) > start and code[start] == '{': + # This is a {{ escape. + added_prefix = prefix + '{{' + start += 1 + continue + + recursion_level += 1 + yield tok(found, prefix=prefix) + added_prefix = '' + + expression = '' + squared_count = 0 + curly_count = 0 + while True: + expr_match = _compiled_in_expr.match(code, start) + expression += expr_match.group(1) + found = expr_match.group(2) + start = expr_match.end() + + if found == '{': + curly_count += 1 + expression += found + elif found == '}' and curly_count > 0: + curly_count -= 1 + expression += found + elif found == '[': + squared_count += 1 + expression += found + elif found == ']': + # Use a max function here, because the Python code might + # just have syntax errors. + squared_count = max(0, squared_count - 1) + expression += found + elif found == ':' and (squared_count or curly_count): + expression += found + elif found in ('"', "'"): + search = found + if len(code) > start + 1 and \ + code[start] == found == code[start+1]: + search *= 3 + start += 2 + + index = code.find(search, start) + if index == -1: + yield tok(expression, type=TokenNamespace.PYTHON_EXPR) + yield tok( + found + code[start:], + type=TokenNamespace.UNTERMINATED_STRING, + ) + start = len(code) + break + expression += found + code[start:index+1] + start = index + 1 + elif found == '!' and len(code) > start and code[start] == '=': + # This is a python `!=` and not a conversion. + expression += found + else: + yield tok(expression, type=TokenNamespace.PYTHON_EXPR) + if found: + yield tok(found) + break + + if found == '!': + conversion_match = _compiled_conversion.match(code, start) + found = conversion_match.group(2) + start = conversion_match.end() + yield tok(conversion_match.group(1), type=TokenNamespace.CONVERSION) + if found: + yield tok(found) + if found == '}': + recursion_level -= 1 + + # We don't need to handle everything after ':', because that is + # basically new tokens. + + yield tok('', type=TokenNamespace.ENDMARKER, prefix=prefix) + + +class Parser(parser.BaseParser): + def parse(self, tokens): + node = super(Parser, self).parse(tokens) + if isinstance(node, self.default_leaf): # Is an endmarker. + # If there's no curly braces we get back a non-module. We always + # want an fstring. + node = self.default_node('fstring', [node]) + + return node + + def convert_leaf(self, pgen_grammar, type, value, prefix, start_pos): + # TODO this is so ugly. + leaf_type = TokenNamespace.token_map[type].lower() + return TypedLeaf(leaf_type, value, start_pos, prefix) + + def error_recovery(self, pgen_grammar, stack, arcs, typ, value, start_pos, prefix, + add_token_callback): + if not self._error_recovery: + return super(Parser, self).error_recovery( + pgen_grammar, stack, arcs, typ, value, start_pos, prefix, + add_token_callback + ) + + token_type = TokenNamespace.token_map[typ].lower() + if len(stack) == 1: + error_leaf = ErrorLeaf(token_type, value, start_pos, prefix) + stack[0][2][1].append(error_leaf) + else: + dfa, state, (type_, nodes) = stack[1] + stack[0][2][1].append(ErrorNode(nodes)) + stack[1:] = [] + + add_token_callback(typ, value, start_pos, prefix) diff --git a/pythonFiles/release/jedi/parser/grammar2.7.txt b/pythonFiles/parso/python/grammar26.txt old mode 100755 new mode 100644 similarity index 60% rename from pythonFiles/release/jedi/parser/grammar2.7.txt rename to pythonFiles/parso/python/grammar26.txt index b29501436b5c..b972a41d6a4a --- a/pythonFiles/release/jedi/parser/grammar2.7.txt +++ b/pythonFiles/parso/python/grammar26.txt @@ -1,4 +1,4 @@ -# Grammar for 2to3. This grammar supports Python 2.x and 3.x. +# Grammar for Python # Note: Changing the grammar specified in this file will most likely # require corresponding changes in the parser module @@ -10,41 +10,44 @@ # NOTE WELL: You should also follow all the steps listed in PEP 306, # "How to Change Python's Grammar" +# Commands for Kees Blom's railroad program +#diagram:token NAME +#diagram:token NUMBER +#diagram:token STRING +#diagram:token NEWLINE +#diagram:token ENDMARKER +#diagram:token INDENT +#diagram:output\input python.bla +#diagram:token DEDENT +#diagram:output\textwidth 20.04cm\oddsidemargin 0.0cm\evensidemargin 0.0cm +#diagram:rules # Start symbols for the grammar: -# file_input is a module or sequence of commands read from an input file; -# single_input is a single interactive statement; -# eval_input is the input for the eval() and input() functions. +# single_input is a single interactive statement; +# file_input is a module or sequence of commands read from an input file; +# eval_input is the input for the eval() and input() functions. # NB: compound_stmt in single_input is followed by extra NEWLINE! -file_input: (NEWLINE | stmt)* ENDMARKER single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE +file_input: (NEWLINE | stmt)* ENDMARKER eval_input: testlist NEWLINE* ENDMARKER decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE decorators: decorator+ decorated: decorators (classdef | funcdef) -funcdef: 'def' NAME parameters ['->' test] ':' suite -parameters: '(' [typedargslist] ')' -typedargslist: ((tfpdef ['=' test] ',')* - ('*' [tname] (',' tname ['=' test])* [',' '**' tname] | '**' tname) - | tfpdef ['=' test] (',' tfpdef ['=' test])* [',']) -tname: NAME [':' test] -tfpdef: tname | '(' tfplist ')' -tfplist: tfpdef (',' tfpdef)* [','] -varargslist: ((vfpdef ['=' test] ',')* - ('*' [vname] (',' vname ['=' test])* [',' '**' vname] | '**' vname) - | vfpdef ['=' test] (',' vfpdef ['=' test])* [',']) -vname: NAME -vfpdef: vname | '(' vfplist ')' -vfplist: vfpdef (',' vfpdef)* [','] +funcdef: 'def' NAME parameters ':' suite +parameters: '(' [varargslist] ')' +varargslist: ((fpdef ['=' test] ',')* + ('*' NAME [',' '**' NAME] | '**' NAME) | + fpdef ['=' test] (',' fpdef ['=' test])* [',']) +fpdef: NAME | '(' fplist ')' +fplist: fpdef (',' fpdef)* [','] stmt: simple_stmt | compound_stmt simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE small_stmt: (expr_stmt | print_stmt | del_stmt | pass_stmt | flow_stmt | import_stmt | global_stmt | exec_stmt | assert_stmt) -expr_stmt: testlist_star_expr (augassign (yield_expr|testlist) | - ('=' (yield_expr|testlist_star_expr))*) -testlist_star_expr: (test|star_expr) (',' (test|star_expr))* [','] +expr_stmt: testlist (augassign (yield_expr|testlist) | + ('=' (yield_expr|testlist))*) augassign: ('+=' | '-=' | '*=' | '/=' | '%=' | '&=' | '|=' | '^=' | '<<=' | '>>=' | '**=' | '//=') # For normal assignments, additional restrictions enforced by the interpreter @@ -57,18 +60,17 @@ break_stmt: 'break' continue_stmt: 'continue' return_stmt: 'return' [testlist] yield_stmt: yield_expr -raise_stmt: 'raise' [test ['from' test | ',' test [',' test]]] +raise_stmt: 'raise' [test [',' test [',' test]]] import_stmt: import_name | import_from import_name: 'import' dotted_as_names -# note below: the ('.' | '...') is necessary because '...' is tokenized as ELLIPSIS -import_from: ('from' (('.' | '...')* dotted_name | ('.' | '...')+) +import_from: ('from' ('.'* dotted_name | '.'+) 'import' ('*' | '(' import_as_names ')' | import_as_names)) import_as_name: NAME ['as' NAME] dotted_as_name: dotted_name ['as' NAME] import_as_names: import_as_name (',' import_as_name)* [','] dotted_as_names: dotted_as_name (',' dotted_as_name)* dotted_name: NAME ('.' NAME)* -global_stmt: ('global' | 'nonlocal') NAME (',' NAME)* +global_stmt: 'global' NAME (',' NAME)* exec_stmt: 'exec' expr ['in' test [',' test]] assert_stmt: 'assert' test [',' test] @@ -78,17 +80,17 @@ while_stmt: 'while' test ':' suite ['else' ':' suite] for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite] try_stmt: ('try' ':' suite ((except_clause ':' suite)+ - ['else' ':' suite] - ['finally' ':' suite] | - 'finally' ':' suite)) -with_stmt: 'with' with_item (',' with_item)* ':' suite + ['else' ':' suite] + ['finally' ':' suite] | + 'finally' ':' suite)) +with_stmt: 'with' with_item ':' suite +# Dave: Python2.6 actually defines a little bit of a different label called +# 'with_var'. However in 2.7+ this is the default. Apply it for +# consistency reasons. with_item: test ['as' expr] -with_var: 'as' expr # NB compile.c makes sure that the default except clause is last -except_clause: 'except' [test [(',' | 'as') test]] -# Edit by David Halter: The stmt is now optional. This reflects how Jedi allows -# classes and functions to be empty, which is beneficial for autocompletion. -suite: simple_stmt | NEWLINE INDENT stmt* DEDENT +except_clause: 'except' [test [('as' | ',') test]] +suite: simple_stmt | NEWLINE INDENT stmt+ DEDENT # Backward compatibility cruft to support: # [ x for x in lambda: True, lambda: False if x() ] @@ -105,7 +107,6 @@ and_test: not_test ('and' not_test)* not_test: 'not' not_test | comparison comparison: expr (comp_op expr)* comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not' -star_expr: '*' expr expr: xor_expr ('|' xor_expr)* xor_expr: and_expr ('^' and_expr)* and_expr: shift_expr ('&' shift_expr)* @@ -115,34 +116,39 @@ term: factor (('*'|'/'|'%'|'//') factor)* factor: ('+'|'-'|'~') factor | power power: atom trailer* ['**' factor] atom: ('(' [yield_expr|testlist_comp] ')' | - '[' [testlist_comp] ']' | + '[' [listmaker] ']' | '{' [dictorsetmaker] '}' | '`' testlist1 '`' | - NAME | NUMBER | STRING+ | '.' '.' '.') -# Modification by David Halter, remove `testlist_gexp` and `listmaker` -testlist_comp: (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] ) + NAME | NUMBER | STRING+) +listmaker: test ( list_for | (',' test)* [','] ) +# Dave: Renamed testlist_gexpr to testlist_comp, because in 2.7+ this is the +# default. It's more consistent like this. +testlist_comp: test ( gen_for | (',' test)* [','] ) lambdef: 'lambda' [varargslist] ':' test trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME subscriptlist: subscript (',' subscript)* [','] -subscript: test | [test] ':' [test] [sliceop] +subscript: '.' '.' '.' | test | [test] ':' [test] [sliceop] sliceop: ':' [test] -exprlist: (expr|star_expr) (',' (expr|star_expr))* [','] +exprlist: expr (',' expr)* [','] testlist: test (',' test)* [','] -# Modification by David Halter, dictsetmaker -> dictorsetmaker (so that it's -# the same as in the 3.4 grammar). -dictorsetmaker: ( (test ':' test (comp_for | (',' test ':' test)* [','])) | - (test (comp_for | (',' test)* [','])) ) +# Dave: Rename from dictmaker to dictorsetmaker, because this is more +# consistent with the following grammars. +dictorsetmaker: test ':' test (',' test ':' test)* [','] -classdef: 'class' NAME ['(' [arglist] ')'] ':' suite +classdef: 'class' NAME ['(' [testlist] ')'] ':' suite arglist: (argument ',')* (argument [','] |'*' test (',' argument)* [',' '**' test] |'**' test) -argument: test [comp_for] | test '=' test # Really [keyword '='] test +argument: test [gen_for] | test '=' test # Really [keyword '='] test + +list_iter: list_for | list_if +list_for: 'for' exprlist 'in' testlist_safe [list_iter] +list_if: 'if' old_test [list_iter] -comp_iter: comp_for | comp_if -comp_for: 'for' exprlist 'in' testlist_safe [comp_iter] -comp_if: 'if' old_test [comp_iter] +gen_iter: gen_for | gen_if +gen_for: 'for' exprlist 'in' or_test [gen_iter] +gen_if: 'if' old_test [gen_iter] testlist1: test (',' test)* diff --git a/pythonFiles/preview/jedi/parser/grammar2.7.txt b/pythonFiles/parso/python/grammar27.txt similarity index 65% rename from pythonFiles/preview/jedi/parser/grammar2.7.txt rename to pythonFiles/parso/python/grammar27.txt index 515dea646236..4c3f33da32d5 100644 --- a/pythonFiles/preview/jedi/parser/grammar2.7.txt +++ b/pythonFiles/parso/python/grammar27.txt @@ -1,4 +1,4 @@ -# Grammar for 2to3. This grammar supports Python 2.x and 3.x. +# Grammar for Python # Note: Changing the grammar specified in this file will most likely # require corresponding changes in the parser module @@ -10,41 +10,32 @@ # NOTE WELL: You should also follow all the steps listed in PEP 306, # "How to Change Python's Grammar" - # Start symbols for the grammar: -# file_input is a module or sequence of commands read from an input file; -# single_input is a single interactive statement; -# eval_input is the input for the eval() and input() functions. +# single_input is a single interactive statement; +# file_input is a module or sequence of commands read from an input file; +# eval_input is the input for the eval() and input() functions. # NB: compound_stmt in single_input is followed by extra NEWLINE! -file_input: (NEWLINE | stmt)* ENDMARKER single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE +file_input: (NEWLINE | stmt)* ENDMARKER eval_input: testlist NEWLINE* ENDMARKER decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE decorators: decorator+ decorated: decorators (classdef | funcdef) -funcdef: 'def' NAME parameters ['->' test] ':' suite -parameters: '(' [typedargslist] ')' -typedargslist: ((tfpdef ['=' test] ',')* - ('*' [tname] (',' tname ['=' test])* [',' '**' tname] | '**' tname) - | tfpdef ['=' test] (',' tfpdef ['=' test])* [',']) -tname: NAME [':' test] -tfpdef: tname | '(' tfplist ')' -tfplist: tfpdef (',' tfpdef)* [','] -varargslist: ((vfpdef ['=' test] ',')* - ('*' [vname] (',' vname ['=' test])* [',' '**' vname] | '**' vname) - | vfpdef ['=' test] (',' vfpdef ['=' test])* [',']) -vname: NAME -vfpdef: vname | '(' vfplist ')' -vfplist: vfpdef (',' vfpdef)* [','] +funcdef: 'def' NAME parameters ':' suite +parameters: '(' [varargslist] ')' +varargslist: ((fpdef ['=' test] ',')* + ('*' NAME [',' '**' NAME] | '**' NAME) | + fpdef ['=' test] (',' fpdef ['=' test])* [',']) +fpdef: NAME | '(' fplist ')' +fplist: fpdef (',' fpdef)* [','] stmt: simple_stmt | compound_stmt simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE small_stmt: (expr_stmt | print_stmt | del_stmt | pass_stmt | flow_stmt | import_stmt | global_stmt | exec_stmt | assert_stmt) -expr_stmt: testlist_star_expr (augassign (yield_expr|testlist) | - ('=' (yield_expr|testlist_star_expr))*) -testlist_star_expr: (test|star_expr) (',' (test|star_expr))* [','] +expr_stmt: testlist (augassign (yield_expr|testlist) | + ('=' (yield_expr|testlist))*) augassign: ('+=' | '-=' | '*=' | '/=' | '%=' | '&=' | '|=' | '^=' | '<<=' | '>>=' | '**=' | '//=') # For normal assignments, additional restrictions enforced by the interpreter @@ -60,8 +51,7 @@ yield_stmt: yield_expr raise_stmt: 'raise' [test [',' test [',' test]]] import_stmt: import_name | import_from import_name: 'import' dotted_as_names -# note below: the ('.' | '...') is necessary because '...' is tokenized as ELLIPSIS -import_from: ('from' (('.' | '...')* dotted_name | ('.' | '...')+) +import_from: ('from' ('.'* dotted_name | '.'+) 'import' ('*' | '(' import_as_names ')' | import_as_names)) import_as_name: NAME ['as' NAME] dotted_as_name: dotted_name ['as' NAME] @@ -78,17 +68,14 @@ while_stmt: 'while' test ':' suite ['else' ':' suite] for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite] try_stmt: ('try' ':' suite ((except_clause ':' suite)+ - ['else' ':' suite] - ['finally' ':' suite] | - 'finally' ':' suite)) + ['else' ':' suite] + ['finally' ':' suite] | + 'finally' ':' suite)) with_stmt: 'with' with_item (',' with_item)* ':' suite with_item: test ['as' expr] -with_var: 'as' expr # NB compile.c makes sure that the default except clause is last -except_clause: 'except' [test [(',' | 'as') test]] -# Edit by David Halter: The stmt is now optional. This reflects how Jedi allows -# classes and functions to be empty, which is beneficial for autocompletion. -suite: simple_stmt | NEWLINE INDENT stmt* DEDENT +except_clause: 'except' [test [('as' | ',') test]] +suite: simple_stmt | NEWLINE INDENT stmt+ DEDENT # Backward compatibility cruft to support: # [ x for x in lambda: True, lambda: False if x() ] @@ -105,7 +92,6 @@ and_test: not_test ('and' not_test)* not_test: 'not' not_test | comparison comparison: expr (comp_op expr)* comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not' -star_expr: '*' expr expr: xor_expr ('|' xor_expr)* xor_expr: and_expr ('^' and_expr)* and_expr: shift_expr ('&' shift_expr)* @@ -115,33 +101,37 @@ term: factor (('*'|'/'|'%'|'//') factor)* factor: ('+'|'-'|'~') factor | power power: atom trailer* ['**' factor] atom: ('(' [yield_expr|testlist_comp] ')' | - '[' [testlist_comp] ']' | + '[' [listmaker] ']' | '{' [dictorsetmaker] '}' | '`' testlist1 '`' | - NAME | NUMBER | STRING+ | '.' '.' '.') -# Modification by David Halter, remove `testlist_gexp` and `listmaker` -testlist_comp: (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] ) + NAME | NUMBER | STRING+) +listmaker: test ( list_for | (',' test)* [','] ) +testlist_comp: test ( comp_for | (',' test)* [','] ) lambdef: 'lambda' [varargslist] ':' test trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME subscriptlist: subscript (',' subscript)* [','] -subscript: test | [test] ':' [test] [sliceop] +subscript: '.' '.' '.' | test | [test] ':' [test] [sliceop] sliceop: ':' [test] -exprlist: (expr|star_expr) (',' (expr|star_expr))* [','] +exprlist: expr (',' expr)* [','] testlist: test (',' test)* [','] -# Modification by David Halter, dictsetmaker -> dictorsetmaker (so that it's -# the same as in the 3.4 grammar). dictorsetmaker: ( (test ':' test (comp_for | (',' test ':' test)* [','])) | - (test (comp_for | (',' test)* [','])) ) + (test (comp_for | (',' test)* [','])) ) -classdef: 'class' NAME ['(' [arglist] ')'] ':' suite +classdef: 'class' NAME ['(' [testlist] ')'] ':' suite arglist: (argument ',')* (argument [','] |'*' test (',' argument)* [',' '**' test] |'**' test) -argument: test [comp_for] | test '=' test # Really [keyword '='] test +# The reason that keywords are test nodes instead of NAME is that using NAME +# results in an ambiguity. ast.c makes sure it's a NAME. +argument: test [comp_for] | test '=' test + +list_iter: list_for | list_if +list_for: 'for' exprlist 'in' testlist_safe [list_iter] +list_if: 'if' old_test [list_iter] comp_iter: comp_for | comp_if -comp_for: 'for' exprlist 'in' testlist_safe [comp_iter] +comp_for: 'for' exprlist 'in' or_test [comp_iter] comp_if: 'if' old_test [comp_iter] testlist1: test (',' test)* diff --git a/pythonFiles/release/jedi/parser/grammar3.4.txt b/pythonFiles/parso/python/grammar33.txt old mode 100755 new mode 100644 similarity index 96% rename from pythonFiles/release/jedi/parser/grammar3.4.txt rename to pythonFiles/parso/python/grammar33.txt index d4a32b8e4ee8..d7aaffd60e14 --- a/pythonFiles/release/jedi/parser/grammar3.4.txt +++ b/pythonFiles/parso/python/grammar33.txt @@ -15,8 +15,8 @@ # file_input is a module or sequence of commands read from an input file; # eval_input is the input for the eval() functions. # NB: compound_stmt in single_input is followed by extra NEWLINE! -file_input: (NEWLINE | stmt)* ENDMARKER single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE +file_input: (NEWLINE | stmt)* ENDMARKER eval_input: testlist NEWLINE* ENDMARKER decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE @@ -78,9 +78,7 @@ with_stmt: 'with' with_item (',' with_item)* ':' suite with_item: test ['as' expr] # NB compile.c makes sure that the default except clause is last except_clause: 'except' [test ['as' NAME]] -# Edit by David Halter: The stmt is now optional. This reflects how Jedi allows -# classes and functions to be empty, which is beneficial for autocompletion. -suite: simple_stmt | NEWLINE INDENT stmt* DEDENT +suite: simple_stmt | NEWLINE INDENT stmt+ DEDENT test: or_test ['if' or_test 'else' test] | lambdef test_nocond: or_test | lambdef_nocond diff --git a/pythonFiles/preview/jedi/parser/grammar3.4.txt b/pythonFiles/parso/python/grammar34.txt similarity index 95% rename from pythonFiles/preview/jedi/parser/grammar3.4.txt rename to pythonFiles/parso/python/grammar34.txt index d4a32b8e4ee8..05c3181627db 100644 --- a/pythonFiles/preview/jedi/parser/grammar3.4.txt +++ b/pythonFiles/parso/python/grammar34.txt @@ -7,16 +7,16 @@ # with someone who can; ask around on python-dev for help. Fred # Drake will probably be listening there. -# NOTE WELL: You should also follow all the steps listed in PEP 306, -# "How to Change Python's Grammar" +# NOTE WELL: You should also follow all the steps listed at +# https://docs.python.org/devguide/grammar.html # Start symbols for the grammar: # single_input is a single interactive statement; # file_input is a module or sequence of commands read from an input file; # eval_input is the input for the eval() functions. # NB: compound_stmt in single_input is followed by extra NEWLINE! -file_input: (NEWLINE | stmt)* ENDMARKER single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE +file_input: (NEWLINE | stmt)* ENDMARKER eval_input: testlist NEWLINE* ENDMARKER decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE @@ -78,9 +78,7 @@ with_stmt: 'with' with_item (',' with_item)* ':' suite with_item: test ['as' expr] # NB compile.c makes sure that the default except clause is last except_clause: 'except' [test ['as' NAME]] -# Edit by David Halter: The stmt is now optional. This reflects how Jedi allows -# classes and functions to be empty, which is beneficial for autocompletion. -suite: simple_stmt | NEWLINE INDENT stmt* DEDENT +suite: simple_stmt | NEWLINE INDENT stmt+ DEDENT test: or_test ['if' or_test 'else' test] | lambdef test_nocond: or_test | lambdef_nocond diff --git a/pythonFiles/preview/jedi/parser/grammar3.5.txt b/pythonFiles/parso/python/grammar35.txt similarity index 96% rename from pythonFiles/preview/jedi/parser/grammar3.5.txt rename to pythonFiles/parso/python/grammar35.txt index 96a727187177..c38217f3f97f 100644 --- a/pythonFiles/preview/jedi/parser/grammar3.5.txt +++ b/pythonFiles/parso/python/grammar35.txt @@ -15,8 +15,8 @@ # file_input is a module or sequence of commands read from an input file; # eval_input is the input for the eval() functions. # NB: compound_stmt in single_input is followed by extra NEWLINE! -file_input: (NEWLINE | stmt)* ENDMARKER single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE +file_input: (NEWLINE | stmt)* ENDMARKER eval_input: testlist NEWLINE* ENDMARKER decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE @@ -84,9 +84,7 @@ with_stmt: 'with' with_item (',' with_item)* ':' suite with_item: test ['as' expr] # NB compile.c makes sure that the default except clause is last except_clause: 'except' [test ['as' NAME]] -# Edit by David Halter: The stmt is now optional. This reflects how Jedi allows -# classes and functions to be empty, which is beneficial for autocompletion. -suite: simple_stmt | NEWLINE INDENT stmt* DEDENT +suite: simple_stmt | NEWLINE INDENT stmt+ DEDENT test: or_test ['if' or_test 'else' test] | lambdef test_nocond: or_test | lambdef_nocond @@ -136,7 +134,7 @@ arglist: argument (',' argument)* [','] # to our LL(1) parser. Even though 'test' includes '*expr' in star_expr, # we explicitly match '*' here, too, to give it proper precedence. # Illegal combinations and orderings are blocked in ast.c: -# multiple (test comp_for) arguements are blocked; keyword unpackings +# multiple (test comp_for) arguments are blocked; keyword unpackings # that precede iterable unpackings are blocked; etc. argument: ( test [comp_for] | test '=' test | diff --git a/pythonFiles/preview/jedi/parser/grammar3.6.txt b/pythonFiles/parso/python/grammar36.txt similarity index 91% rename from pythonFiles/preview/jedi/parser/grammar3.6.txt rename to pythonFiles/parso/python/grammar36.txt index b44a56981564..e76147e9e4fc 100644 --- a/pythonFiles/preview/jedi/parser/grammar3.6.txt +++ b/pythonFiles/parso/python/grammar36.txt @@ -1,24 +1,16 @@ # Grammar for Python -# Note: Changing the grammar specified in this file will most likely -# require corresponding changes in the parser module -# (../Modules/parsermodule.c). If you can't make the changes to -# that module yourself, please co-ordinate the required changes -# with someone who can; ask around on python-dev for help. Fred -# Drake will probably be listening there. - # NOTE WELL: You should also follow all the steps listed at # https://docs.python.org/devguide/grammar.html # Start symbols for the grammar: -# file_input is a module or sequence of commands read from an input file; # single_input is a single interactive statement; +# file_input is a module or sequence of commands read from an input file; # eval_input is the input for the eval() functions. # NB: compound_stmt in single_input is followed by extra NEWLINE! -file_input: (NEWLINE | stmt)* ENDMARKER single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE +file_input: (NEWLINE | stmt)* ENDMARKER eval_input: testlist NEWLINE* ENDMARKER - decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE decorators: decorator+ decorated: decorators (classdef | funcdef | async_funcdef) @@ -90,10 +82,7 @@ with_stmt: 'with' with_item (',' with_item)* ':' suite with_item: test ['as' expr] # NB compile.c makes sure that the default except clause is last except_clause: 'except' [test ['as' NAME]] -# Edit by Francisco Souza/David Halter: The stmt is now optional. This reflects -# how Jedi allows classes and functions to be empty, which is beneficial for -# autocompletion. -suite: simple_stmt | NEWLINE INDENT stmt* DEDENT +suite: simple_stmt | NEWLINE INDENT stmt+ DEDENT test: or_test ['if' or_test 'else' test] | lambdef test_nocond: or_test | lambdef_nocond diff --git a/pythonFiles/parso/python/grammar37.txt b/pythonFiles/parso/python/grammar37.txt new file mode 100644 index 000000000000..e76147e9e4fc --- /dev/null +++ b/pythonFiles/parso/python/grammar37.txt @@ -0,0 +1,150 @@ +# Grammar for Python + +# NOTE WELL: You should also follow all the steps listed at +# https://docs.python.org/devguide/grammar.html + +# Start symbols for the grammar: +# single_input is a single interactive statement; +# file_input is a module or sequence of commands read from an input file; +# eval_input is the input for the eval() functions. +# NB: compound_stmt in single_input is followed by extra NEWLINE! +single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE +file_input: (NEWLINE | stmt)* ENDMARKER +eval_input: testlist NEWLINE* ENDMARKER +decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE +decorators: decorator+ +decorated: decorators (classdef | funcdef | async_funcdef) + +# NOTE: Francisco Souza/Reinoud Elhorst, using ASYNC/'await' keywords instead of +# skipping python3.5+ compatibility, in favour of 3.7 solution +async_funcdef: 'async' funcdef +funcdef: 'def' NAME parameters ['->' test] ':' suite + +parameters: '(' [typedargslist] ')' +typedargslist: (tfpdef ['=' test] (',' tfpdef ['=' test])* [',' [ + '*' [tfpdef] (',' tfpdef ['=' test])* [',' ['**' tfpdef [',']]] + | '**' tfpdef [',']]] + | '*' [tfpdef] (',' tfpdef ['=' test])* [',' ['**' tfpdef [',']]] + | '**' tfpdef [',']) +tfpdef: NAME [':' test] +varargslist: (vfpdef ['=' test] (',' vfpdef ['=' test])* [',' [ + '*' [vfpdef] (',' vfpdef ['=' test])* [',' ['**' vfpdef [',']]] + | '**' vfpdef [',']]] + | '*' [vfpdef] (',' vfpdef ['=' test])* [',' ['**' vfpdef [',']]] + | '**' vfpdef [','] +) +vfpdef: NAME + +stmt: simple_stmt | compound_stmt +simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE +small_stmt: (expr_stmt | del_stmt | pass_stmt | flow_stmt | + import_stmt | global_stmt | nonlocal_stmt | assert_stmt) +expr_stmt: testlist_star_expr (annassign | augassign (yield_expr|testlist) | + ('=' (yield_expr|testlist_star_expr))*) +annassign: ':' test ['=' test] +testlist_star_expr: (test|star_expr) (',' (test|star_expr))* [','] +augassign: ('+=' | '-=' | '*=' | '@=' | '/=' | '%=' | '&=' | '|=' | '^=' | + '<<=' | '>>=' | '**=' | '//=') +# For normal and annotated assignments, additional restrictions enforced by the interpreter +del_stmt: 'del' exprlist +pass_stmt: 'pass' +flow_stmt: break_stmt | continue_stmt | return_stmt | raise_stmt | yield_stmt +break_stmt: 'break' +continue_stmt: 'continue' +return_stmt: 'return' [testlist] +yield_stmt: yield_expr +raise_stmt: 'raise' [test ['from' test]] +import_stmt: import_name | import_from +import_name: 'import' dotted_as_names +# note below: the ('.' | '...') is necessary because '...' is tokenized as ELLIPSIS +import_from: ('from' (('.' | '...')* dotted_name | ('.' | '...')+) + 'import' ('*' | '(' import_as_names ')' | import_as_names)) +import_as_name: NAME ['as' NAME] +dotted_as_name: dotted_name ['as' NAME] +import_as_names: import_as_name (',' import_as_name)* [','] +dotted_as_names: dotted_as_name (',' dotted_as_name)* +dotted_name: NAME ('.' NAME)* +global_stmt: 'global' NAME (',' NAME)* +nonlocal_stmt: 'nonlocal' NAME (',' NAME)* +assert_stmt: 'assert' test [',' test] + +compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated | async_stmt +async_stmt: 'async' (funcdef | with_stmt | for_stmt) +if_stmt: 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite] +while_stmt: 'while' test ':' suite ['else' ':' suite] +for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite] +try_stmt: ('try' ':' suite + ((except_clause ':' suite)+ + ['else' ':' suite] + ['finally' ':' suite] | + 'finally' ':' suite)) +with_stmt: 'with' with_item (',' with_item)* ':' suite +with_item: test ['as' expr] +# NB compile.c makes sure that the default except clause is last +except_clause: 'except' [test ['as' NAME]] +suite: simple_stmt | NEWLINE INDENT stmt+ DEDENT + +test: or_test ['if' or_test 'else' test] | lambdef +test_nocond: or_test | lambdef_nocond +lambdef: 'lambda' [varargslist] ':' test +lambdef_nocond: 'lambda' [varargslist] ':' test_nocond +or_test: and_test ('or' and_test)* +and_test: not_test ('and' not_test)* +not_test: 'not' not_test | comparison +comparison: expr (comp_op expr)* +# <> isn't actually a valid comparison operator in Python. It's here for the +# sake of a __future__ import described in PEP 401 (which really works :-) +comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not' +star_expr: '*' expr +expr: xor_expr ('|' xor_expr)* +xor_expr: and_expr ('^' and_expr)* +and_expr: shift_expr ('&' shift_expr)* +shift_expr: arith_expr (('<<'|'>>') arith_expr)* +arith_expr: term (('+'|'-') term)* +term: factor (('*'|'@'|'/'|'%'|'//') factor)* +factor: ('+'|'-'|'~') factor | power +power: atom_expr ['**' factor] +atom_expr: ['await'] atom trailer* +atom: ('(' [yield_expr|testlist_comp] ')' | + '[' [testlist_comp] ']' | + '{' [dictorsetmaker] '}' | + NAME | NUMBER | STRING+ | '...' | 'None' | 'True' | 'False') +testlist_comp: (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] ) +trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME +subscriptlist: subscript (',' subscript)* [','] +subscript: test | [test] ':' [test] [sliceop] +sliceop: ':' [test] +exprlist: (expr|star_expr) (',' (expr|star_expr))* [','] +testlist: test (',' test)* [','] +dictorsetmaker: ( ((test ':' test | '**' expr) + (comp_for | (',' (test ':' test | '**' expr))* [','])) | + ((test | star_expr) + (comp_for | (',' (test | star_expr))* [','])) ) + +classdef: 'class' NAME ['(' [arglist] ')'] ':' suite + +arglist: argument (',' argument)* [','] + +# The reason that keywords are test nodes instead of NAME is that using NAME +# results in an ambiguity. ast.c makes sure it's a NAME. +# "test '=' test" is really "keyword '=' test", but we have no such token. +# These need to be in a single rule to avoid grammar that is ambiguous +# to our LL(1) parser. Even though 'test' includes '*expr' in star_expr, +# we explicitly match '*' here, too, to give it proper precedence. +# Illegal combinations and orderings are blocked in ast.c: +# multiple (test comp_for) arguments are blocked; keyword unpackings +# that precede iterable unpackings are blocked; etc. +argument: ( test [comp_for] | + test '=' test | + '**' test | + '*' test ) + +comp_iter: comp_for | comp_if +comp_for: ['async'] 'for' exprlist 'in' or_test [comp_iter] +comp_if: 'if' test_nocond [comp_iter] + +# not used in grammar, but may appear in "node" passed from Parser to Compiler +encoding_decl: NAME + +yield_expr: 'yield' [yield_arg] +yield_arg: 'from' test | testlist diff --git a/pythonFiles/parso/python/issue_list.txt b/pythonFiles/parso/python/issue_list.txt new file mode 100644 index 000000000000..e5e2c9dda764 --- /dev/null +++ b/pythonFiles/parso/python/issue_list.txt @@ -0,0 +1,176 @@ +A list of syntax/indentation errors I've encountered in CPython. + +# Python/compile.c + "'continue' not properly in loop" + "'continue' not supported inside 'finally' clause" # Until loop + "default 'except:' must be last" + "from __future__ imports must occur at the beginning of the file" + "'return' outside function" + "'return' with value in async generator" + "'break' outside loop" + "two starred expressions in assignment" + "asynchronous comprehension outside of an asynchronous function" + "'yield' outside function" # For both yield and yield from + "'yield from' inside async function" + "'await' outside function" + "'await' outside async function" + "starred assignment target must be in a list or tuple" + "can't use starred expression here" + "too many statically nested blocks" # Max. 20 + # This is one of the few places in the cpython code base that I really + # don't understand. It feels a bit hacky if you look at the implementation + # of UNPACK_EX. + "too many expressions in star-unpacking assignment" + + # Just ignore this one, newer versions will not be affected anymore and + # it's a limit of 2^16 - 1. + "too many annotations" # Only python 3.0 - 3.5, 3.6 is not affected. + +# Python/ast.c + # used with_item exprlist expr_stmt + "can't %s %s" % ("assign to" or "delete", + "lambda" + "function call" # foo() + "generator expression" + "list comprehension" + "set comprehension" + "dict comprehension" + "keyword" + "Ellipsis" + "comparison" + Dict: Set: Num: Str: Bytes: JoinedStr: FormattedValue: + "literal" + BoolOp: BinOp: UnaryOp: + "operator" + Yield: YieldFrom: + "yield expression" + Await: + "await expression" + IfExp: + "conditional expression" + "assignment to keyword" # (keywords + __debug__) # None = 2 + "named arguments must follow bare *" # def foo(*): pass + "non-default argument follows default argument" # def f(x=3, y): pass + "iterable unpacking cannot be used in comprehension" # [*[] for a in [1]] + "dict unpacking cannot be used in dict comprehension" # {**{} for a in [1]} + "Generator expression must be parenthesized if not sole argument" # foo(x for x in [], b) + "positional argument follows keyword argument unpacking" # f(**x, y) >= 3.5 + "positional argument follows keyword argument" # f(x=2, y) >= 3.5 + "iterable argument unpacking follows keyword argument unpacking" # foo(**kwargs, *args) + "lambda cannot contain assignment" # f(lambda: 1=1) + "keyword can't be an expression" # f(+x=1) + "keyword argument repeated" # f(x=1, x=2) + "illegal expression for augmented assignment" # x, y += 1 + "only single target (not list) can be annotated" # [x, y]: int + "only single target (not tuple) can be annotated" # x, y: str + "illegal target for annotation" # True: 1` + "trailing comma not allowed without surrounding parentheses" # from foo import a, + "bytes can only contain ASCII literal characters." # b'ä' # prob. only python 3 + "cannot mix bytes and nonbytes literals" # 's' b'' + "assignment to yield expression not possible" # x = yield 1 = 3 + + "f-string: empty expression not allowed" # f'{}' + "f-string: single '}' is not allowed" # f'}' + "f-string: expressions nested too deeply" # f'{1:{5:{3}}}' + "f-string expression part cannot include a backslash" # f'{"\"}' or f'{"\\"}' + "f-string expression part cannot include '#'" # f'{#}' + "f-string: unterminated string" # f'{"}' + "f-string: mismatched '(', '{', or '['" + "f-string: invalid conversion character: expected 's', 'r', or 'a'" # f'{1!b}' + "f-string: unexpected end of string" # Doesn't really happen?! + "f-string: expecting '}'" # f'{' + "(unicode error) unknown error + "(value error) unknown error + "(unicode error) MESSAGE + MESSAGES = { + "\\ at end of string" + "truncated \\xXX escape" + "truncated \\uXXXX escape" + "truncated \\UXXXXXXXX escape" + "illegal Unicode character" # '\Uffffffff' + "malformed \\N character escape" # '\N{}' + "unknown Unicode character name" # '\N{foo}' + } + "(value error) MESSAGE # bytes + MESSAGES = { + "Trailing \\ in string" + "invalid \\x escape at position %d" + } + + "invalid escape sequence \\%c" # Only happens when used in `python -W error` + "unexpected node" # Probably irrelevant + "Unexpected node-type in from-import" # Irrelevant, doesn't happen. + "malformed 'try' statement" # Irrelevant, doesn't happen. + +# Python/symtable.c + "duplicate argument '%U' in function definition" + "name '%U' is assigned to before global declaration" + "name '%U' is assigned to before nonlocal declaration" + "name '%U' is used prior to global declaration" + "name '%U' is used prior to nonlocal declaration" + "annotated name '%U' can't be global" + "annotated name '%U' can't be nonlocal" + "import * only allowed at module level" + + "name '%U' is parameter and global", + "name '%U' is nonlocal and global", + "name '%U' is parameter and nonlocal", + + "nonlocal declaration not allowed at module level"); + "no binding for nonlocal '%U' found", + # RecursionError. Not handled. For all human written code, this is probably + # not an issue. eval("()"*x) with x>=2998 for example fails, but that's + # more than 2000 executions on one line. + "maximum recursion depth exceeded during compilation"); + +# Python/future.c + "not a chance" + "future feature %.100s is not defined" + "from __future__ imports must occur at the beginning of the file" # Also in compile.c + +# Parser/tokenizer.c + # All the following issues seem to be irrelevant for parso, because the + # encoding stuff is done before it reaches the tokenizer. It's already + # unicode at that point. + "encoding problem: %s" + "encoding problem: %s with BOM" + "Non-UTF-8 code starting with '\\x%.2x' in file %U on line %i, but no encoding declared; see http://python.org/dev/peps/pep-0263/ for details" + +# Parser/pythonrun.c + E_SYNTAX: "invalid syntax" + E_LINECONT: "unexpected character after line continuation character" + E_IDENTIFIER: "invalid character in identifier" + # Also just use 'invalid syntax'. Happens mostly with stuff like `(`. This + # message doesn't really help the user, because it only appears very + # randomly, e.g. `(or` wouldn't yield this error. + E_EOF: "unexpected EOF while parsing" + # Even in 3.6 this is implemented kind of shaky. Not implemented, I think + # cPython needs to fix this one first. + # e.g. `ast.parse('def x():\n\t if 1:\n \t \tpass')` works :/ + E_TABSPACE: "inconsistent use of tabs and spaces in indentation" + # Ignored, just shown as "invalid syntax". The error has mostly to do with + # numbers like 0b2 everywhere or 1.6_ in Python3.6. + E_TOKEN: "invalid token" + E_EOFS: "EOF while scanning triple-quoted string literal" + E_EOLS: "EOL while scanning string literal" + + # IndentationError + E_DEDENT: "unindent does not match any outer indentation level" + E_TOODEEP: "too many levels of indentation" # 100 levels + E_SYNTAX: "expected an indented block" + "unexpected indent" + # I don't think this actually ever happens. + "unexpected unindent" + + + # Irrelevant for parso for now. + E_OVERFLOW: "expression too long" + E_DECODE: "unknown decode error" + E_BADSINGLE: "multiple statements found while compiling a single statement" + + +Version specific: +Python 3.5: + 'yield' inside async function +Python 3.3/3.4: + can use starred expression only as assignment target diff --git a/pythonFiles/parso/python/parser.py b/pythonFiles/parso/python/parser.py new file mode 100644 index 000000000000..1897f53e8d6f --- /dev/null +++ b/pythonFiles/parso/python/parser.py @@ -0,0 +1,261 @@ +from parso.python import tree +from parso.python.token import (DEDENT, INDENT, ENDMARKER, NEWLINE, NUMBER, + STRING, tok_name, NAME) +from parso.parser import BaseParser +from parso.pgen2.parse import token_to_ilabel + + +class Parser(BaseParser): + """ + This class is used to parse a Python file, it then divides them into a + class structure of different scopes. + + :param pgen_grammar: The grammar object of pgen2. Loaded by load_grammar. + """ + + node_map = { + 'expr_stmt': tree.ExprStmt, + 'classdef': tree.Class, + 'funcdef': tree.Function, + 'file_input': tree.Module, + 'import_name': tree.ImportName, + 'import_from': tree.ImportFrom, + 'break_stmt': tree.KeywordStatement, + 'continue_stmt': tree.KeywordStatement, + 'return_stmt': tree.ReturnStmt, + 'raise_stmt': tree.KeywordStatement, + 'yield_expr': tree.YieldExpr, + 'del_stmt': tree.KeywordStatement, + 'pass_stmt': tree.KeywordStatement, + 'global_stmt': tree.GlobalStmt, + 'nonlocal_stmt': tree.KeywordStatement, + 'print_stmt': tree.KeywordStatement, + 'assert_stmt': tree.AssertStmt, + 'if_stmt': tree.IfStmt, + 'with_stmt': tree.WithStmt, + 'for_stmt': tree.ForStmt, + 'while_stmt': tree.WhileStmt, + 'try_stmt': tree.TryStmt, + 'comp_for': tree.CompFor, + # Not sure if this is the best idea, but IMO it's the easiest way to + # avoid extreme amounts of work around the subtle difference of 2/3 + # grammar in list comoprehensions. + 'list_for': tree.CompFor, + # Same here. This just exists in Python 2.6. + 'gen_for': tree.CompFor, + 'decorator': tree.Decorator, + 'lambdef': tree.Lambda, + 'old_lambdef': tree.Lambda, + 'lambdef_nocond': tree.Lambda, + } + default_node = tree.PythonNode + + def __init__(self, pgen_grammar, error_recovery=True, start_symbol='file_input'): + super(Parser, self).__init__(pgen_grammar, start_symbol, error_recovery=error_recovery) + + self.syntax_errors = [] + self._omit_dedent_list = [] + self._indent_counter = 0 + + # TODO do print absolute import detection here. + # try: + # del python_grammar_no_print_statement.keywords["print"] + # except KeyError: + # pass # Doesn't exist in the Python 3 grammar. + + # if self.options["print_function"]: + # python_grammar = pygram.python_grammar_no_print_statement + # else: + + def parse(self, tokens): + if self._error_recovery: + if self._start_symbol != 'file_input': + raise NotImplementedError + + tokens = self._recovery_tokenize(tokens) + + node = super(Parser, self).parse(tokens) + + if self._start_symbol == 'file_input' != node.type: + # If there's only one statement, we get back a non-module. That's + # not what we want, we want a module, so we add it here: + node = self.convert_node( + self._pgen_grammar, + self._pgen_grammar.symbol2number['file_input'], + [node] + ) + + return node + + def convert_node(self, pgen_grammar, type, children): + """ + Convert raw node information to a PythonBaseNode instance. + + This is passed to the parser driver which calls it whenever a reduction of a + grammar rule produces a new complete node, so that the tree is build + strictly bottom-up. + """ + # TODO REMOVE symbol, we don't want type here. + symbol = pgen_grammar.number2symbol[type] + try: + return self.node_map[symbol](children) + except KeyError: + if symbol == 'suite': + # We don't want the INDENT/DEDENT in our parser tree. Those + # leaves are just cancer. They are virtual leaves and not real + # ones and therefore have pseudo start/end positions and no + # prefixes. Just ignore them. + children = [children[0]] + children[2:-1] + elif symbol == 'list_if': + # Make transitioning from 2 to 3 easier. + symbol = 'comp_if' + elif symbol == 'listmaker': + # Same as list_if above. + symbol = 'testlist_comp' + return self.default_node(symbol, children) + + def convert_leaf(self, pgen_grammar, type, value, prefix, start_pos): + # print('leaf', repr(value), token.tok_name[type]) + if type == NAME: + if value in pgen_grammar.keywords: + return tree.Keyword(value, start_pos, prefix) + else: + return tree.Name(value, start_pos, prefix) + elif type == STRING: + return tree.String(value, start_pos, prefix) + elif type == NUMBER: + return tree.Number(value, start_pos, prefix) + elif type == NEWLINE: + return tree.Newline(value, start_pos, prefix) + elif type == ENDMARKER: + return tree.EndMarker(value, start_pos, prefix) + else: + return tree.Operator(value, start_pos, prefix) + + def error_recovery(self, pgen_grammar, stack, arcs, typ, value, start_pos, prefix, + add_token_callback): + def get_symbol_and_nodes(stack): + for dfa, state, (type_, nodes) in stack: + symbol = pgen_grammar.number2symbol[type_] + yield symbol, nodes + + tos_nodes = stack.get_tos_nodes() + if tos_nodes: + last_leaf = tos_nodes[-1].get_last_leaf() + else: + last_leaf = None + + if self._start_symbol == 'file_input' and \ + (typ == ENDMARKER or typ == DEDENT and '\n' not in last_leaf.value): + def reduce_stack(states, newstate): + # reduce + state = newstate + while states[state] == [(0, state)]: + self.pgen_parser._pop() + + dfa, state, (type_, nodes) = stack[-1] + states, first = dfa + + + # In Python statements need to end with a newline. But since it's + # possible (and valid in Python ) that there's no newline at the + # end of a file, we have to recover even if the user doesn't want + # error recovery. + #print('x', pprint.pprint(stack)) + ilabel = token_to_ilabel(pgen_grammar, NEWLINE, value) + + dfa, state, (type_, nodes) = stack[-1] + symbol = pgen_grammar.number2symbol[type_] + states, first = dfa + arcs = states[state] + # Look for a state with this label + for i, newstate in arcs: + if ilabel == i: + if symbol == 'simple_stmt': + # This is basically shifting + stack[-1] = (dfa, newstate, (type_, nodes)) + + reduce_stack(states, newstate) + add_token_callback(typ, value, start_pos, prefix) + return + # Check if we're at the right point + #for symbol, nodes in get_symbol_and_nodes(stack): + # self.pgen_parser._pop() + + #break + break + #symbol = pgen_grammar.number2symbol[type_] + + if not self._error_recovery: + return super(Parser, self).error_recovery( + pgen_grammar, stack, arcs, typ, value, start_pos, prefix, + add_token_callback) + + def current_suite(stack): + # For now just discard everything that is not a suite or + # file_input, if we detect an error. + for index, (symbol, nodes) in reversed(list(enumerate(get_symbol_and_nodes(stack)))): + # `suite` can sometimes be only simple_stmt, not stmt. + if symbol == 'file_input': + break + elif symbol == 'suite' and len(nodes) > 1: + # suites without an indent in them get discarded. + break + return index, symbol, nodes + + index, symbol, nodes = current_suite(stack) + + # print('err', token.tok_name[typ], repr(value), start_pos, len(stack), index) + if self._stack_removal(pgen_grammar, stack, arcs, index + 1, value, start_pos): + add_token_callback(typ, value, start_pos, prefix) + else: + if typ == INDENT: + # For every deleted INDENT we have to delete a DEDENT as well. + # Otherwise the parser will get into trouble and DEDENT too early. + self._omit_dedent_list.append(self._indent_counter) + + error_leaf = tree.PythonErrorLeaf(tok_name[typ].lower(), value, start_pos, prefix) + stack[-1][2][1].append(error_leaf) + + if symbol == 'suite': + dfa, state, node = stack[-1] + states, first = dfa + arcs = states[state] + intended_label = pgen_grammar.symbol2label['stmt'] + # Introduce a proper state transition. We're basically allowing + # there to be no valid statements inside a suite. + if [x[0] for x in arcs] == [intended_label]: + new_state = arcs[0][1] + stack[-1] = dfa, new_state, node + + def _stack_removal(self, pgen_grammar, stack, arcs, start_index, value, start_pos): + failed_stack = False + found = False + all_nodes = [] + for dfa, state, (type_, nodes) in stack[start_index:]: + if nodes: + found = True + if found: + failed_stack = True + all_nodes += nodes + if failed_stack: + stack[start_index - 1][2][1].append(tree.PythonErrorNode(all_nodes)) + + stack[start_index:] = [] + return failed_stack + + def _recovery_tokenize(self, tokens): + for typ, value, start_pos, prefix in tokens: + # print(tok_name[typ], repr(value), start_pos, repr(prefix)) + if typ == DEDENT: + # We need to count indents, because if we just omit any DEDENT, + # we might omit them in the wrong place. + o = self._omit_dedent_list + if o and o[-1] == self._indent_counter: + o.pop() + continue + + self._indent_counter -= 1 + elif typ == INDENT: + self._indent_counter += 1 + yield typ, value, start_pos, prefix diff --git a/pythonFiles/parso/python/pep8.py b/pythonFiles/parso/python/pep8.py new file mode 100644 index 000000000000..59fe452d06c4 --- /dev/null +++ b/pythonFiles/parso/python/pep8.py @@ -0,0 +1,727 @@ +import re +from contextlib import contextmanager + +from parso.python.errors import ErrorFinder, ErrorFinderConfig +from parso.normalizer import Rule +from parso.python.tree import search_ancestor, Flow, Scope + + +_IMPORT_TYPES = ('import_name', 'import_from') +_SUITE_INTRODUCERS = ('classdef', 'funcdef', 'if_stmt', 'while_stmt', + 'for_stmt', 'try_stmt', 'with_stmt') +_NON_STAR_TYPES = ('term', 'import_from', 'power') +_OPENING_BRACKETS = '(', '[', '{' +_CLOSING_BRACKETS = ')', ']', '}' +_FACTOR = '+', '-', '~' +_ALLOW_SPACE = '*', '+', '-', '**', '/', '//', '@' +_BITWISE_OPERATOR = '<<', '>>', '|', '&', '^' +_NEEDS_SPACE = ('=', '%', '->', + '<', '>', '==', '>=', '<=', '<>', '!=', + '+=', '-=', '*=', '@=', '/=', '%=', '&=', '|=', '^=', '<<=', + '>>=', '**=', '//=') +_NEEDS_SPACE += _BITWISE_OPERATOR +_IMPLICIT_INDENTATION_TYPES = ('dictorsetmaker', 'argument') +_POSSIBLE_SLICE_PARENTS = ('subscript', 'subscriptlist', 'sliceop') + + +class IndentationTypes(object): + VERTICAL_BRACKET = object() + HANGING_BRACKET = object() + BACKSLASH = object() + SUITE = object() + IMPLICIT = object() + + +class IndentationNode(object): + type = IndentationTypes.SUITE + + def __init__(self, config, indentation, parent=None): + self.bracket_indentation = self.indentation = indentation + self.parent = parent + + def __repr__(self): + return '<%s>' % self.__class__.__name__ + + def get_latest_suite_node(self): + n = self + while n is not None: + if n.type == IndentationTypes.SUITE: + return n + + n = n.parent + + +class BracketNode(IndentationNode): + def __init__(self, config, leaf, parent, in_suite_introducer=False): + self.leaf = leaf + + # Figure out here what the indentation is. For chained brackets + # we can basically use the previous indentation. + previous_leaf = leaf + n = parent + if n.type == IndentationTypes.IMPLICIT: + n = n.parent + while True: + if hasattr(n, 'leaf') and previous_leaf.line != n.leaf.line: + break + + previous_leaf = previous_leaf.get_previous_leaf() + if not isinstance(n, BracketNode) or previous_leaf != n.leaf: + break + n = n.parent + parent_indentation = n.indentation + + + next_leaf = leaf.get_next_leaf() + if '\n' in next_leaf.prefix: + # This implies code like: + # foobarbaz( + # a, + # b, + # ) + self.bracket_indentation = parent_indentation \ + + config.closing_bracket_hanging_indentation + self.indentation = parent_indentation + config.indentation + self.type = IndentationTypes.HANGING_BRACKET + else: + # Implies code like: + # foobarbaz( + # a, + # b, + # ) + expected_end_indent = leaf.end_pos[1] + if '\t' in config.indentation: + self.indentation = None + else: + self.indentation = ' ' * expected_end_indent + self.bracket_indentation = self.indentation + self.type = IndentationTypes.VERTICAL_BRACKET + + if in_suite_introducer and parent.type == IndentationTypes.SUITE \ + and self.indentation == parent_indentation + config.indentation: + self.indentation += config.indentation + # The closing bracket should have the same indentation. + self.bracket_indentation = self.indentation + self.parent = parent + + +class ImplicitNode(BracketNode): + """ + Implicit indentation after keyword arguments, default arguments, + annotations and dict values. + """ + def __init__(self, config, leaf, parent): + super(ImplicitNode, self).__init__(config, leaf, parent) + self.type = IndentationTypes.IMPLICIT + + next_leaf = leaf.get_next_leaf() + if leaf == ':' and '\n' not in next_leaf.prefix: + self.indentation += ' ' + + +class BackslashNode(IndentationNode): + type = IndentationTypes.BACKSLASH + + def __init__(self, config, parent_indentation, containing_leaf, spacing, parent=None): + expr_stmt = search_ancestor(containing_leaf, 'expr_stmt') + if expr_stmt is not None: + equals = expr_stmt.children[-2] + + if '\t' in config.indentation: + # TODO unite with the code of BracketNode + self.indentation = None + else: + # If the backslash follows the equals, use normal indentation + # otherwise it should align with the equals. + if equals.end_pos == spacing.start_pos: + self.indentation = parent_indentation + config.indentation + else: + # +1 because there is a space. + self.indentation = ' ' * (equals.end_pos[1] + 1) + else: + self.indentation = parent_indentation + config.indentation + self.bracket_indentation = self.indentation + self.parent = parent + + +def _is_magic_name(name): + return name.value.startswith('__') and name.value.endswith('__') + + +class PEP8Normalizer(ErrorFinder): + def __init__(self, *args, **kwargs): + super(PEP8Normalizer, self).__init__(*args, **kwargs) + self._previous_part = None + self._previous_leaf = None + self._on_newline = True + self._newline_count = 0 + self._wanted_newline_count = None + self._max_new_lines_in_prefix = 0 + self._new_statement = True + self._implicit_indentation_possible = False + # The top of stack of the indentation nodes. + self._indentation_tos = self._last_indentation_tos = \ + IndentationNode(self._config, indentation='') + self._in_suite_introducer = False + + if ' ' in self._config.indentation: + self._indentation_type = 'spaces' + self._wrong_indentation_char = '\t' + else: + self._indentation_type = 'tabs' + self._wrong_indentation_char = ' ' + + @contextmanager + def visit_node(self, node): + with super(PEP8Normalizer, self).visit_node(node): + with self._visit_node(node): + yield + + @contextmanager + def _visit_node(self, node): + typ = node.type + + if typ in 'import_name': + names = node.get_defined_names() + if len(names) > 1: + for name in names[:1]: + self.add_issue(name, 401, 'Multiple imports on one line') + elif typ == 'lambdef': + expr_stmt = node.parent + # Check if it's simply defining a single name, not something like + # foo.bar or x[1], where using a lambda could make more sense. + if expr_stmt.type == 'expr_stmt' and any(n.type == 'name' for n in expr_stmt.children[:-2:2]): + self.add_issue(node, 731, 'Do not assign a lambda expression, use a def') + elif typ == 'try_stmt': + for child in node.children: + # Here we can simply check if it's an except, because otherwise + # it would be an except_clause. + if child.type == 'keyword' and child.value == 'except': + self.add_issue(child, 722, 'Do not use bare except, specify exception instead') + elif typ == 'comparison': + for child in node.children: + if child.type not in ('atom_expr', 'power'): + continue + if len(child.children) > 2: + continue + trailer = child.children[1] + atom = child.children[0] + if trailer.type == 'trailer' and atom.type == 'name' \ + and atom.value == 'type': + self.add_issue(node, 721, "Do not compare types, use 'isinstance()") + break + elif typ == 'file_input': + endmarker = node.children[-1] + prev = endmarker.get_previous_leaf() + prefix = endmarker.prefix + if (not prefix.endswith('\n') and ( + prefix or prev is None or prev.value != '\n')): + self.add_issue(endmarker, 292, "No newline at end of file") + + if typ in _IMPORT_TYPES: + simple_stmt = node.parent + module = simple_stmt.parent + #if module.type == 'simple_stmt': + if module.type == 'file_input': + index = module.children.index(simple_stmt) + for child in module.children[:index]: + children = [child] + if child.type == 'simple_stmt': + # Remove the newline. + children = child.children[:-1] + + found_docstring = False + for c in children: + if c.type == 'string' and not found_docstring: + continue + found_docstring = True + + if c.type == 'expr_stmt' and \ + all(_is_magic_name(n) for n in c.get_defined_names()): + continue + + if c.type in _IMPORT_TYPES or isinstance(c, Flow): + continue + + self.add_issue(node, 402, 'Module level import not at top of file') + break + else: + continue + break + + implicit_indentation_possible = typ in _IMPLICIT_INDENTATION_TYPES + in_introducer = typ in _SUITE_INTRODUCERS + if in_introducer: + self._in_suite_introducer = True + elif typ == 'suite': + if self._indentation_tos.type == IndentationTypes.BACKSLASH: + self._indentation_tos = self._indentation_tos.parent + + self._indentation_tos = IndentationNode( + self._config, + self._indentation_tos.indentation + self._config.indentation, + parent=self._indentation_tos + ) + elif implicit_indentation_possible: + self._implicit_indentation_possible = True + yield + if typ == 'suite': + assert self._indentation_tos.type == IndentationTypes.SUITE + self._indentation_tos = self._indentation_tos.parent + # If we dedent, no lines are needed anymore. + self._wanted_newline_count = None + elif implicit_indentation_possible: + self._implicit_indentation_possible = False + if self._indentation_tos.type == IndentationTypes.IMPLICIT: + self._indentation_tos = self._indentation_tos.parent + elif in_introducer: + self._in_suite_introducer = False + if typ in ('classdef', 'funcdef'): + self._wanted_newline_count = self._get_wanted_blank_lines_count() + + def _check_tabs_spaces(self, spacing): + if self._wrong_indentation_char in spacing.value: + self.add_issue(spacing, 101, 'Indentation contains ' + self._indentation_type) + return True + return False + + def _get_wanted_blank_lines_count(self): + suite_node = self._indentation_tos.get_latest_suite_node() + return int(suite_node.parent is None) + 1 + + def _reset_newlines(self, spacing, leaf, is_comment=False): + self._max_new_lines_in_prefix = \ + max(self._max_new_lines_in_prefix, self._newline_count) + + wanted = self._wanted_newline_count + if wanted is not None: + # Need to substract one + blank_lines = self._newline_count - 1 + if wanted > blank_lines and leaf.type != 'endmarker': + # In case of a comment we don't need to add the issue, yet. + if not is_comment: + # TODO end_pos wrong. + code = 302 if wanted == 2 else 301 + message = "expected %s blank line, found %s" \ + % (wanted, blank_lines) + self.add_issue(spacing, code, message) + self._wanted_newline_count = None + else: + self._wanted_newline_count = None + + if not is_comment: + wanted = self._get_wanted_blank_lines_count() + actual = self._max_new_lines_in_prefix - 1 + + val = leaf.value + needs_lines = ( + val == '@' and leaf.parent.type == 'decorator' + or ( + val == 'class' + or val == 'async' and leaf.get_next_leaf() == 'def' + or val == 'def' and self._previous_leaf != 'async' + ) and leaf.parent.parent.type != 'decorated' + ) + if needs_lines and actual < wanted: + func_or_cls = leaf.parent + suite = func_or_cls.parent + if suite.type == 'decorated': + suite = suite.parent + + # The first leaf of a file or a suite should not need blank + # lines. + if suite.children[int(suite.type == 'suite')] != func_or_cls: + code = 302 if wanted == 2 else 301 + message = "expected %s blank line, found %s" \ + % (wanted, actual) + self.add_issue(spacing, code, message) + + self._max_new_lines_in_prefix = 0 + + self._newline_count = 0 + + def visit_leaf(self, leaf): + super(PEP8Normalizer, self).visit_leaf(leaf) + for part in leaf._split_prefix(): + if part.type == 'spacing': + # This part is used for the part call after for. + break + self._visit_part(part, part.create_spacing_part(), leaf) + + self._analyse_non_prefix(leaf) + self._visit_part(leaf, part, leaf) + + # Cleanup + self._last_indentation_tos = self._indentation_tos + + self._new_statement = leaf.type == 'newline' + + # TODO does this work? with brackets and stuff? + if leaf.type == 'newline' and \ + self._indentation_tos.type == IndentationTypes.BACKSLASH: + self._indentation_tos = self._indentation_tos.parent + + if leaf.value == ':' and leaf.parent.type in _SUITE_INTRODUCERS: + self._in_suite_introducer = False + elif leaf.value == 'elif': + self._in_suite_introducer = True + + if not self._new_statement: + self._reset_newlines(part, leaf) + self._max_blank_lines = 0 + + self._previous_leaf = leaf + + return leaf.value + + def _visit_part(self, part, spacing, leaf): + value = part.value + type_ = part.type + if type_ == 'error_leaf': + return + + if value == ',' and part.parent.type == 'dictorsetmaker': + self._indentation_tos = self._indentation_tos.parent + + node = self._indentation_tos + + if type_ == 'comment': + if value.startswith('##'): + # Whole blocks of # should not raise an error. + if value.lstrip('#'): + self.add_issue(part, 266, "Too many leading '#' for block comment.") + elif self._on_newline: + if not re.match('#:? ', value) and not value == '#' \ + and not (value.startswith('#!') and part.start_pos == (1, 0)): + self.add_issue(part, 265, "Block comment should start with '# '") + else: + if not re.match('#:? [^ ]', value): + self.add_issue(part, 262, "Inline comment should start with '# '") + + self._reset_newlines(spacing, leaf, is_comment=True) + elif type_ == 'newline': + if self._newline_count > self._get_wanted_blank_lines_count(): + self.add_issue(part, 303, "Too many blank lines (%s)" % self._newline_count) + elif leaf in ('def', 'class') \ + and leaf.parent.parent.type == 'decorated': + self.add_issue(part, 304, "Blank lines found after function decorator") + + + self._newline_count += 1 + + if type_ == 'backslash': + # TODO is this enough checking? What about ==? + if node.type != IndentationTypes.BACKSLASH: + if node.type != IndentationTypes.SUITE: + self.add_issue(part, 502, 'The backslash is redundant between brackets') + else: + indentation = node.indentation + if self._in_suite_introducer and node.type == IndentationTypes.SUITE: + indentation += self._config.indentation + + self._indentation_tos = BackslashNode( + self._config, + indentation, + part, + spacing, + parent=self._indentation_tos + ) + elif self._on_newline: + indentation = spacing.value + if node.type == IndentationTypes.BACKSLASH \ + and self._previous_part.type == 'newline': + self._indentation_tos = self._indentation_tos.parent + + if not self._check_tabs_spaces(spacing): + should_be_indentation = node.indentation + if type_ == 'comment': + # Comments can be dedented. So we have to care for that. + n = self._last_indentation_tos + while True: + if len(indentation) > len(n.indentation): + break + + should_be_indentation = n.indentation + + self._last_indentation_tos = n + if n == node: + break + n = n.parent + + if self._new_statement: + if type_ == 'newline': + if indentation: + self.add_issue(spacing, 291, 'Trailing whitespace') + elif indentation != should_be_indentation: + s = '%s %s' % (len(self._config.indentation), self._indentation_type) + self.add_issue(part, 111, 'Indentation is not a multiple of ' + s) + else: + if value in '])}': + should_be_indentation = node.bracket_indentation + else: + should_be_indentation = node.indentation + if self._in_suite_introducer and indentation == \ + node.get_latest_suite_node().indentation \ + + self._config.indentation: + self.add_issue(part, 129, "Line with same indent as next logical block") + elif indentation != should_be_indentation: + if not self._check_tabs_spaces(spacing) and part.value != '\n': + if value in '])}': + if node.type == IndentationTypes.VERTICAL_BRACKET: + self.add_issue(part, 124, "Closing bracket does not match visual indentation") + else: + self.add_issue(part, 123, "Losing bracket does not match indentation of opening bracket's line") + else: + if len(indentation) < len(should_be_indentation): + if node.type == IndentationTypes.VERTICAL_BRACKET: + self.add_issue(part, 128, 'Continuation line under-indented for visual indent') + elif node.type == IndentationTypes.BACKSLASH: + self.add_issue(part, 122, 'Continuation line missing indentation or outdented') + elif node.type == IndentationTypes.IMPLICIT: + self.add_issue(part, 135, 'xxx') + else: + self.add_issue(part, 121, 'Continuation line under-indented for hanging indent') + else: + if node.type == IndentationTypes.VERTICAL_BRACKET: + self.add_issue(part, 127, 'Continuation line over-indented for visual indent') + elif node.type == IndentationTypes.IMPLICIT: + self.add_issue(part, 136, 'xxx') + else: + self.add_issue(part, 126, 'Continuation line over-indented for hanging indent') + else: + self._check_spacing(part, spacing) + + self._check_line_length(part, spacing) + # ------------------------------- + # Finalizing. Updating the state. + # ------------------------------- + if value and value in '()[]{}' and type_ != 'error_leaf' \ + and part.parent.type != 'error_node': + if value in _OPENING_BRACKETS: + self._indentation_tos = BracketNode( + self._config, part, + parent=self._indentation_tos, + in_suite_introducer=self._in_suite_introducer + ) + else: + assert node.type != IndentationTypes.IMPLICIT + self._indentation_tos = self._indentation_tos.parent + elif value in ('=', ':') and self._implicit_indentation_possible \ + and part.parent.type in _IMPLICIT_INDENTATION_TYPES: + indentation = node.indentation + self._indentation_tos = ImplicitNode( + self._config, part, parent=self._indentation_tos + ) + + self._on_newline = type_ in ('newline', 'backslash', 'bom') + + self._previous_part = part + self._previous_spacing = spacing + + def _check_line_length(self, part, spacing): + if part.type == 'backslash': + last_column = part.start_pos[1] + 1 + else: + last_column = part.end_pos[1] + if last_column > self._config.max_characters \ + and spacing.start_pos[1] <= self._config.max_characters : + # Special case for long URLs in multi-line docstrings or comments, + # but still report the error when the 72 first chars are whitespaces. + report = True + if part.type == 'comment': + splitted = part.value[1:].split() + if len(splitted) == 1 \ + and (part.end_pos[1] - len(splitted[0])) < 72: + report = False + if report: + self.add_issue( + part, + 501, + 'Line too long (%s > %s characters)' % + (last_column, self._config.max_characters), + ) + + def _check_spacing(self, part, spacing): + def add_if_spaces(*args): + if spaces: + return self.add_issue(*args) + + def add_not_spaces(*args): + if not spaces: + return self.add_issue(*args) + + spaces = spacing.value + prev = self._previous_part + if prev is not None and prev.type == 'error_leaf' or part.type == 'error_leaf': + return + + type_ = part.type + if '\t' in spaces: + self.add_issue(spacing, 223, 'Used tab to separate tokens') + elif type_ == 'comment': + if len(spaces) < self._config.spaces_before_comment: + self.add_issue(spacing, 261, 'At least two spaces before inline comment') + elif type_ == 'newline': + add_if_spaces(spacing, 291, 'Trailing whitespace') + elif len(spaces) > 1: + self.add_issue(spacing, 221, 'Multiple spaces used') + else: + if prev in _OPENING_BRACKETS: + message = "Whitespace after '%s'" % part.value + add_if_spaces(spacing, 201, message) + elif part in _CLOSING_BRACKETS: + message = "Whitespace before '%s'" % part.value + add_if_spaces(spacing, 202, message) + elif part in (',', ';') or part == ':' \ + and part.parent.type not in _POSSIBLE_SLICE_PARENTS: + message = "Whitespace before '%s'" % part.value + add_if_spaces(spacing, 203, message) + elif prev == ':' and prev.parent.type in _POSSIBLE_SLICE_PARENTS: + pass # TODO + elif prev in (',', ';', ':'): + add_not_spaces(spacing, 231, "missing whitespace after '%s'") + elif part == ':': # Is a subscript + # TODO + pass + elif part in ('*', '**') and part.parent.type not in _NON_STAR_TYPES \ + or prev in ('*', '**') \ + and prev.parent.type not in _NON_STAR_TYPES: + # TODO + pass + elif prev in _FACTOR and prev.parent.type == 'factor': + pass + elif prev == '@' and prev.parent.type == 'decorator': + pass # TODO should probably raise an error if there's a space here + elif part in _NEEDS_SPACE or prev in _NEEDS_SPACE: + if part == '=' and part.parent.type in ('argument', 'param') \ + or prev == '=' and prev.parent.type in ('argument', 'param'): + if part == '=': + param = part.parent + else: + param = prev.parent + if param.type == 'param' and param.annotation: + add_not_spaces(spacing, 252, 'Expected spaces around annotation equals') + else: + add_if_spaces(spacing, 251, 'Unexpected spaces around keyword / parameter equals') + elif part in _BITWISE_OPERATOR or prev in _BITWISE_OPERATOR: + add_not_spaces(spacing, 227, 'Missing whitespace around bitwise or shift operator') + elif part == '%' or prev == '%': + add_not_spaces(spacing, 228, 'Missing whitespace around modulo operator') + else: + message_225 = 'Missing whitespace between tokens' + add_not_spaces(spacing, 225, message_225) + elif type_ == 'keyword' or prev.type == 'keyword': + add_not_spaces(spacing, 275, 'Missing whitespace around keyword') + else: + prev_spacing = self._previous_spacing + if prev in _ALLOW_SPACE and spaces != prev_spacing.value \ + and '\n' not in self._previous_leaf.prefix: + message = "Whitespace before operator doesn't match with whitespace after" + self.add_issue(spacing, 229, message) + + if spaces and part not in _ALLOW_SPACE and prev not in _ALLOW_SPACE: + message_225 = 'Missing whitespace between tokens' + #print('xy', spacing) + #self.add_issue(spacing, 225, message_225) + # TODO why only brackets? + if part in _OPENING_BRACKETS: + message = "Whitespace before '%s'" % part.value + add_if_spaces(spacing, 211, message) + + def _analyse_non_prefix(self, leaf): + typ = leaf.type + if typ == 'name' and leaf.value in ('l', 'O', 'I'): + if leaf.is_definition(): + message = "Do not define %s named 'l', 'O', or 'I' one line" + if leaf.parent.type == 'class' and leaf.parent.name == leaf: + self.add_issue(leaf, 742, message % 'classes') + elif leaf.parent.type == 'function' and leaf.parent.name == leaf: + self.add_issue(leaf, 743, message % 'function') + else: + self.add_issuadd_issue(741, message % 'variables', leaf) + elif leaf.value == ':': + if isinstance(leaf.parent, (Flow, Scope)) and leaf.parent.type != 'lambdef': + next_leaf = leaf.get_next_leaf() + if next_leaf.type != 'newline': + if leaf.parent.type == 'funcdef': + self.add_issue(next_leaf, 704, 'Multiple statements on one line (def)') + else: + self.add_issue(next_leaf, 701, 'Multiple statements on one line (colon)') + elif leaf.value == ';': + if leaf.get_next_leaf().type in ('newline', 'endmarker'): + self.add_issue(leaf, 703, 'Statement ends with a semicolon') + else: + self.add_issue(leaf, 702, 'Multiple statements on one line (semicolon)') + elif leaf.value in ('==', '!='): + comparison = leaf.parent + index = comparison.children.index(leaf) + left = comparison.children[index - 1] + right = comparison.children[index + 1] + for node in left, right: + if node.type == 'keyword' or node.type == 'name': + if node.value == 'None': + message = "comparison to None should be 'if cond is None:'" + self.add_issue(leaf, 711, message) + break + elif node.value in ('True', 'False'): + message = "comparison to False/True should be 'if cond is True:' or 'if cond:'" + self.add_issue(leaf, 712, message) + break + elif leaf.value in ('in', 'is'): + comparison = leaf.parent + if comparison.type == 'comparison' and comparison.parent.type == 'not_test': + if leaf.value == 'in': + self.add_issue(leaf, 713, "test for membership should be 'not in'") + else: + self.add_issue(leaf, 714, "test for object identity should be 'is not'") + elif typ == 'string': + # Checking multiline strings + for i, line in enumerate(leaf.value.splitlines()[1:]): + indentation = re.match('[ \t]*', line).group(0) + start_pos = leaf.line + i, len(indentation) + # TODO check multiline indentation. + elif typ == 'endmarker': + if self._newline_count >= 2: + self.add_issue(leaf, 391, 'Blank line at end of file') + + def add_issue(self, node, code, message): + if self._previous_leaf is not None: + if search_ancestor(self._previous_leaf, 'error_node') is not None: + return + if self._previous_leaf.type == 'error_leaf': + return + if search_ancestor(node, 'error_node') is not None: + return + if code in (901, 903): + # 901 and 903 are raised by the ErrorFinder. + super(PEP8Normalizer, self).add_issue(node, code, message) + else: + # Skip ErrorFinder here, because it has custom behavior. + super(ErrorFinder, self).add_issue(node, code, message) + + +class PEP8NormalizerConfig(ErrorFinderConfig): + normalizer_class = PEP8Normalizer + """ + Normalizing to PEP8. Not really implemented, yet. + """ + def __init__(self, indentation=' ' * 4, hanging_indentation=None, + max_characters=79, spaces_before_comment=2): + self.indentation = indentation + if hanging_indentation is None: + hanging_indentation = indentation + self.hanging_indentation = hanging_indentation + self.closing_bracket_hanging_indentation = '' + self.break_after_binary = False + self.max_characters = max_characters + self.spaces_before_comment = spaces_before_comment + + +# TODO this is not yet ready. +#@PEP8Normalizer.register_rule(type='endmarker') +class BlankLineAtEnd(Rule): + code = 392 + message = 'Blank line at end of file' + + def is_issue(self, leaf): + return self._newline_count >= 2 diff --git a/pythonFiles/parso/python/prefix.py b/pythonFiles/parso/python/prefix.py new file mode 100644 index 000000000000..b7f1e1bc4db9 --- /dev/null +++ b/pythonFiles/parso/python/prefix.py @@ -0,0 +1,97 @@ +import re +from codecs import BOM_UTF8 + +from parso.python.tokenize import group + +unicode_bom = BOM_UTF8.decode('utf-8') + + +class PrefixPart(object): + def __init__(self, leaf, typ, value, spacing='', start_pos=None): + assert start_pos is not None + self.parent = leaf + self.type = typ + self.value = value + self.spacing = spacing + self.start_pos = start_pos + + @property + def end_pos(self): + if self.value.endswith('\n'): + return self.start_pos[0] + 1, 0 + if self.value == unicode_bom: + # The bom doesn't have a length at the start of a Python file. + return self.start_pos + return self.start_pos[0], self.start_pos[1] + len(self.value) + + def create_spacing_part(self): + column = self.start_pos[1] - len(self.spacing) + return PrefixPart( + self.parent, 'spacing', self.spacing, + start_pos=(self.start_pos[0], column) + ) + + def __repr__(self): + return '%s(%s, %s, %s)' % ( + self.__class__.__name__, + self.type, + repr(self.value), + self.start_pos + ) + + +_comment = r'#[^\n\r\f]*' +_backslash = r'\\\r?\n' +_newline = r'\r?\n' +_form_feed = r'\f' +_only_spacing = '$' +_spacing = r'[ \t]*' +_bom = unicode_bom + +_regex = group( + _comment, _backslash, _newline, _form_feed, _only_spacing, _bom, + capture=True +) +_regex = re.compile(group(_spacing, capture=True) + _regex) + + +_types = { + '#': 'comment', + '\\': 'backslash', + '\f': 'formfeed', + '\n': 'newline', + '\r': 'newline', + unicode_bom: 'bom' +} + + +def split_prefix(leaf, start_pos): + line, column = start_pos + start = 0 + value = spacing = '' + bom = False + while start != len(leaf.prefix): + match =_regex.match(leaf.prefix, start) + spacing = match.group(1) + value = match.group(2) + if not value: + break + type_ = _types[value[0]] + yield PrefixPart( + leaf, type_, value, spacing, + start_pos=(line, column + start - int(bom) + len(spacing)) + ) + if type_ == 'bom': + bom = True + + start = match.end(0) + if value.endswith('\n'): + line += 1 + column = -start + + if value: + spacing = '' + yield PrefixPart( + leaf, 'spacing', spacing, + start_pos=(line, column + start) + ) diff --git a/pythonFiles/preview/jedi/parser/token.py b/pythonFiles/parso/python/token.py similarity index 56% rename from pythonFiles/preview/jedi/parser/token.py rename to pythonFiles/parso/python/token.py index 0cb846da11a3..fb590a5f28c6 100644 --- a/pythonFiles/preview/jedi/parser/token.py +++ b/pythonFiles/parso/python/token.py @@ -1,34 +1,36 @@ from __future__ import absolute_import - -from jedi._compatibility import is_py3, is_py35 +from itertools import count from token import * +from parso._compatibility import py_version + + +_counter = count(N_TOKENS) +# Never want to see this thing again. +del N_TOKENS -COMMENT = N_TOKENS +COMMENT = next(_counter) tok_name[COMMENT] = 'COMMENT' -N_TOKENS += 1 -NL = N_TOKENS +NL = next(_counter) tok_name[NL] = 'NL' -N_TOKENS += 1 -if is_py3: - BACKQUOTE = N_TOKENS +# Sets the attributes that don't exist in these tok_name versions. +if py_version >= 30: + BACKQUOTE = next(_counter) tok_name[BACKQUOTE] = 'BACKQUOTE' - N_TOKENS += 1 else: - RARROW = N_TOKENS + RARROW = next(_counter) tok_name[RARROW] = 'RARROW' - N_TOKENS += 1 - ELLIPSIS = N_TOKENS + ELLIPSIS = next(_counter) tok_name[ELLIPSIS] = 'ELLIPSIS' - N_TOKENS += 1 -if not is_py35: - ATEQUAL = N_TOKENS +if py_version < 35: + ATEQUAL = next(_counter) tok_name[ATEQUAL] = 'ATEQUAL' - N_TOKENS += 1 +ERROR_DEDENT = next(_counter) +tok_name[ERROR_DEDENT] = 'ERROR_DEDENT' # Map from operator to number (since tokenize doesn't do this) @@ -88,3 +90,15 @@ for line in opmap_raw.splitlines(): op, name = line.split() opmap[op] = globals()[name] + + +def generate_token_id(string): + """ + Uses a token in the grammar (e.g. `'+'` or `'and'`returns the corresponding + ID for it. The strings are part of the grammar file. + """ + try: + return opmap[string] + except KeyError: + pass + return globals()[string] diff --git a/pythonFiles/parso/python/tokenize.py b/pythonFiles/parso/python/tokenize.py new file mode 100644 index 000000000000..ecd2437f5ebb --- /dev/null +++ b/pythonFiles/parso/python/tokenize.py @@ -0,0 +1,420 @@ +# -*- coding: utf-8 -*- +""" +This tokenizer has been copied from the ``tokenize.py`` standard library +tokenizer. The reason was simple: The standard library tokenizer fails +if the indentation is not right. To make it possible to do error recovery the + tokenizer needed to be rewritten. + +Basically this is a stripped down version of the standard library module, so +you can read the documentation there. Additionally we included some speed and +memory optimizations here. +""" +from __future__ import absolute_import + +import sys +import string +import re +from collections import namedtuple +import itertools as _itertools +from codecs import BOM_UTF8 + +from parso.python.token import (tok_name, ENDMARKER, STRING, NUMBER, opmap, + NAME, ERRORTOKEN, NEWLINE, INDENT, DEDENT, + ERROR_DEDENT) +from parso._compatibility import py_version +from parso.utils import split_lines + + +TokenCollection = namedtuple( + 'TokenCollection', + 'pseudo_token single_quoted triple_quoted endpats always_break_tokens', +) + +BOM_UTF8_STRING = BOM_UTF8.decode('utf-8') + +_token_collection_cache = {} + +if py_version >= 30: + # Python 3 has str.isidentifier() to check if a char is a valid identifier + is_identifier = str.isidentifier +else: + namechars = string.ascii_letters + '_' + is_identifier = lambda s: s in namechars + + +def group(*choices, **kwargs): + capture = kwargs.pop('capture', False) # Python 2, arrghhhhh :( + assert not kwargs + + start = '(' + if not capture: + start += '?:' + return start + '|'.join(choices) + ')' + + +def any(*choices): + return group(*choices) + '*' + + +def maybe(*choices): + return group(*choices) + '?' + + +# Return the empty string, plus all of the valid string prefixes. +def _all_string_prefixes(version_info): + def different_case_versions(prefix): + for s in _itertools.product(*[(c, c.upper()) for c in prefix]): + yield ''.join(s) + # The valid string prefixes. Only contain the lower case versions, + # and don't contain any permuations (include 'fr', but not + # 'rf'). The various permutations will be generated. + _valid_string_prefixes = ['b', 'r', 'u'] + if version_info >= (3, 0): + _valid_string_prefixes.append('br') + + if version_info >= (3, 6): + _valid_string_prefixes += ['f', 'fr'] + + # if we add binary f-strings, add: ['fb', 'fbr'] + result = set(['']) + for prefix in _valid_string_prefixes: + for t in _itertools.permutations(prefix): + # create a list with upper and lower versions of each + # character + result.update(different_case_versions(t)) + if version_info <= (2, 7): + # In Python 2 the order cannot just be random. + result.update(different_case_versions('ur')) + result.update(different_case_versions('br')) + return result + + +def _compile(expr): + return re.compile(expr, re.UNICODE) + + +def _get_token_collection(version_info): + try: + return _token_collection_cache[tuple(version_info)] + except KeyError: + _token_collection_cache[tuple(version_info)] = result = \ + _create_token_collection(version_info) + return result + + +def _create_token_collection(version_info): + # Note: we use unicode matching for names ("\w") but ascii matching for + # number literals. + Whitespace = r'[ \f\t]*' + Comment = r'#[^\r\n]*' + Name = r'\w+' + + if version_info >= (3, 6): + Hexnumber = r'0[xX](?:_?[0-9a-fA-F])+' + Binnumber = r'0[bB](?:_?[01])+' + Octnumber = r'0[oO](?:_?[0-7])+' + Decnumber = r'(?:0(?:_?0)*|[1-9](?:_?[0-9])*)' + Intnumber = group(Hexnumber, Binnumber, Octnumber, Decnumber) + Exponent = r'[eE][-+]?[0-9](?:_?[0-9])*' + Pointfloat = group(r'[0-9](?:_?[0-9])*\.(?:[0-9](?:_?[0-9])*)?', + r'\.[0-9](?:_?[0-9])*') + maybe(Exponent) + Expfloat = r'[0-9](?:_?[0-9])*' + Exponent + Floatnumber = group(Pointfloat, Expfloat) + Imagnumber = group(r'[0-9](?:_?[0-9])*[jJ]', Floatnumber + r'[jJ]') + else: + Hexnumber = r'0[xX][0-9a-fA-F]+' + Binnumber = r'0[bB][01]+' + if version_info >= (3, 0): + Octnumber = r'0[oO][0-7]+' + else: + Octnumber = '0[oO]?[0-7]+' + Decnumber = r'(?:0+|[1-9][0-9]*)' + Intnumber = group(Hexnumber, Binnumber, Octnumber, Decnumber) + Exponent = r'[eE][-+]?[0-9]+' + Pointfloat = group(r'[0-9]+\.[0-9]*', r'\.[0-9]+') + maybe(Exponent) + Expfloat = r'[0-9]+' + Exponent + Floatnumber = group(Pointfloat, Expfloat) + Imagnumber = group(r'[0-9]+[jJ]', Floatnumber + r'[jJ]') + Number = group(Imagnumber, Floatnumber, Intnumber) + + # Note that since _all_string_prefixes includes the empty string, + # StringPrefix can be the empty string (making it optional). + possible_prefixes = _all_string_prefixes(version_info) + StringPrefix = group(*possible_prefixes) + + # Tail end of ' string. + Single = r"[^'\\]*(?:\\.[^'\\]*)*'" + # Tail end of " string. + Double = r'[^"\\]*(?:\\.[^"\\]*)*"' + # Tail end of ''' string. + Single3 = r"[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''" + # Tail end of """ string. + Double3 = r'[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""' + Triple = group(StringPrefix + "'''", StringPrefix + '"""') + + # Because of leftmost-then-longest match semantics, be sure to put the + # longest operators first (e.g., if = came before ==, == would get + # recognized as two instances of =). + Operator = group(r"\*\*=?", r">>=?", r"<<=?", r"!=", + r"//=?", r"->", + r"[+\-*/%&@`|^=<>]=?", + r"~") + + Bracket = '[][(){}]' + + special_args = [r'\r?\n', r'[:;.,@]'] + if version_info >= (3, 0): + special_args.insert(0, r'\.\.\.') + Special = group(*special_args) + + Funny = group(Operator, Bracket, Special) + + # First (or only) line of ' or " string. + ContStr = group(StringPrefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*" + + group("'", r'\\\r?\n'), + StringPrefix + r'"[^\n"\\]*(?:\\.[^\n"\\]*)*' + + group('"', r'\\\r?\n')) + PseudoExtras = group(r'\\\r?\n|\Z', Comment, Triple) + PseudoToken = group(Whitespace, capture=True) + \ + group(PseudoExtras, Number, Funny, ContStr, Name, capture=True) + + # For a given string prefix plus quotes, endpats maps it to a regex + # to match the remainder of that string. _prefix can be empty, for + # a normal single or triple quoted string (with no prefix). + endpats = {} + for _prefix in possible_prefixes: + endpats[_prefix + "'"] = _compile(Single) + endpats[_prefix + '"'] = _compile(Double) + endpats[_prefix + "'''"] = _compile(Single3) + endpats[_prefix + '"""'] = _compile(Double3) + + # A set of all of the single and triple quoted string prefixes, + # including the opening quotes. + single_quoted = set() + triple_quoted = set() + for t in possible_prefixes: + for p in (t + '"', t + "'"): + single_quoted.add(p) + for p in (t + '"""', t + "'''"): + triple_quoted.add(p) + + ALWAYS_BREAK_TOKENS = (';', 'import', 'class', 'def', 'try', 'except', + 'finally', 'while', 'with', 'return') + pseudo_token_compiled = _compile(PseudoToken) + return TokenCollection( + pseudo_token_compiled, single_quoted, triple_quoted, endpats, + ALWAYS_BREAK_TOKENS + ) + + +class Token(namedtuple('Token', ['type', 'string', 'start_pos', 'prefix'])): + @property + def end_pos(self): + lines = split_lines(self.string) + if len(lines) > 1: + return self.start_pos[0] + len(lines) - 1, 0 + else: + return self.start_pos[0], self.start_pos[1] + len(self.string) + + +class PythonToken(Token): + def _get_type_name(self, exact=True): + return tok_name[self.type] + + def __repr__(self): + return ('TokenInfo(type=%s, string=%r, start=%r, prefix=%r)' % + self._replace(type=self._get_type_name())) + + +def tokenize(code, version_info, start_pos=(1, 0)): + """Generate tokens from a the source code (string).""" + lines = split_lines(code, keepends=True) + return tokenize_lines(lines, version_info, start_pos=start_pos) + + +def tokenize_lines(lines, version_info, start_pos=(1, 0)): + """ + A heavily modified Python standard library tokenizer. + + Additionally to the default information, yields also the prefix of each + token. This idea comes from lib2to3. The prefix contains all information + that is irrelevant for the parser like newlines in parentheses or comments. + """ + pseudo_token, single_quoted, triple_quoted, endpats, always_break_tokens, = \ + _get_token_collection(version_info) + paren_level = 0 # count parentheses + indents = [0] + max = 0 + numchars = '0123456789' + contstr = '' + contline = None + # We start with a newline. This makes indent at the first position + # possible. It's not valid Python, but still better than an INDENT in the + # second line (and not in the first). This makes quite a few things in + # Jedi's fast parser possible. + new_line = True + prefix = '' # Should never be required, but here for safety + additional_prefix = '' + first = True + lnum = start_pos[0] - 1 + for line in lines: # loop over lines in stream + lnum += 1 + pos = 0 + max = len(line) + if first: + if line.startswith(BOM_UTF8_STRING): + additional_prefix = BOM_UTF8_STRING + line = line[1:] + max = len(line) + + # Fake that the part before was already parsed. + line = '^' * start_pos[1] + line + pos = start_pos[1] + max += start_pos[1] + + first = False + + if contstr: # continued string + endmatch = endprog.match(line) + if endmatch: + pos = endmatch.end(0) + yield PythonToken(STRING, contstr + line[:pos], contstr_start, prefix) + contstr = '' + contline = None + else: + contstr = contstr + line + contline = contline + line + continue + + while pos < max: + pseudomatch = pseudo_token.match(line, pos) + if not pseudomatch: # scan for tokens + txt = line[pos:] + if txt.endswith('\n'): + new_line = True + yield PythonToken(ERRORTOKEN, txt, (lnum, pos), additional_prefix) + additional_prefix = '' + break + + prefix = additional_prefix + pseudomatch.group(1) + additional_prefix = '' + start, pos = pseudomatch.span(2) + spos = (lnum, start) + token = pseudomatch.group(2) + if token == '': + assert prefix + additional_prefix = prefix + # This means that we have a line with whitespace/comments at + # the end, which just results in an endmarker. + break + initial = token[0] + + if new_line and initial not in '\r\n#': + new_line = False + if paren_level == 0: + i = 0 + while line[i] == '\f': + i += 1 + start -= 1 + if start > indents[-1]: + yield PythonToken(INDENT, '', spos, '') + indents.append(start) + while start < indents[-1]: + if start > indents[-2]: + yield PythonToken(ERROR_DEDENT, '', (lnum, 0), '') + break + yield PythonToken(DEDENT, '', spos, '') + indents.pop() + + if (initial in numchars or # ordinary number + (initial == '.' and token != '.' and token != '...')): + yield PythonToken(NUMBER, token, spos, prefix) + elif initial in '\r\n': + if not new_line and paren_level == 0: + yield PythonToken(NEWLINE, token, spos, prefix) + else: + additional_prefix = prefix + token + new_line = True + elif initial == '#': # Comments + assert not token.endswith("\n") + additional_prefix = prefix + token + elif token in triple_quoted: + endprog = endpats[token] + endmatch = endprog.match(line, pos) + if endmatch: # all on one line + pos = endmatch.end(0) + token = line[start:pos] + yield PythonToken(STRING, token, spos, prefix) + else: + contstr_start = (lnum, start) # multiple lines + contstr = line[start:] + contline = line + break + elif initial in single_quoted or \ + token[:2] in single_quoted or \ + token[:3] in single_quoted: + if token[-1] == '\n': # continued string + contstr_start = lnum, start + endprog = (endpats.get(initial) or endpats.get(token[1]) + or endpats.get(token[2])) + contstr = line[start:] + contline = line + break + else: # ordinary string + yield PythonToken(STRING, token, spos, prefix) + elif is_identifier(initial): # ordinary name + if token in always_break_tokens: + paren_level = 0 + while True: + indent = indents.pop() + if indent > start: + yield PythonToken(DEDENT, '', spos, '') + else: + indents.append(indent) + break + yield PythonToken(NAME, token, spos, prefix) + elif initial == '\\' and line[start:] in ('\\\n', '\\\r\n'): # continued stmt + additional_prefix += prefix + line[start:] + break + else: + if token in '([{': + paren_level += 1 + elif token in ')]}': + paren_level -= 1 + + try: + # This check is needed in any case to check if it's a valid + # operator or just some random unicode character. + typ = opmap[token] + except KeyError: + typ = ERRORTOKEN + yield PythonToken(typ, token, spos, prefix) + + if contstr: + yield PythonToken(ERRORTOKEN, contstr, contstr_start, prefix) + if contstr.endswith('\n'): + new_line = True + + end_pos = lnum, max + # As the last position we just take the maximally possible position. We + # remove -1 for the last new line. + for indent in indents[1:]: + yield PythonToken(DEDENT, '', end_pos, '') + yield PythonToken(ENDMARKER, '', end_pos, additional_prefix) + + +if __name__ == "__main__": + if len(sys.argv) >= 2: + path = sys.argv[1] + with open(path) as f: + code = f.read() + else: + code = sys.stdin.read() + + from parso.utils import python_bytes_to_unicode, parse_version_string + + if isinstance(code, bytes): + code = python_bytes_to_unicode(code) + + for token in tokenize(code, parse_version_string()): + print(token) diff --git a/pythonFiles/parso/python/tree.py b/pythonFiles/parso/python/tree.py new file mode 100644 index 000000000000..eb977800a607 --- /dev/null +++ b/pythonFiles/parso/python/tree.py @@ -0,0 +1,1165 @@ +""" +This is the syntax tree for Python syntaxes (2 & 3). The classes represent +syntax elements like functions and imports. + +All of the nodes can be traced back to the `Python grammar file +`_. If you want to know how +a tree is structured, just analyse that file (for each Python version it's a +bit different). + +There's a lot of logic here that makes it easier for Jedi (and other libraries) +to deal with a Python syntax tree. + +By using :py:meth:`parso.tree.NodeOrLeaf.get_code` on a module, you can get +back the 1-to-1 representation of the input given to the parser. This is +important if you want to refactor a parser tree. + +>>> from parso import parse +>>> parser = parse('import os') +>>> module = parser.get_root_node() +>>> module + + +Any subclasses of :class:`Scope`, including :class:`Module` has an attribute +:attr:`iter_imports `: + +>>> list(module.iter_imports()) +[] + +Changes to the Python Grammar +----------------------------- + +A few things have changed when looking at Python grammar files: + +- :class:`Param` does not exist in Python grammar files. It is essentially a + part of a ``parameters`` node. |parso| splits it up to make it easier to + analyse parameters. However this just makes it easier to deal with the syntax + tree, it doesn't actually change the valid syntax. +- A few nodes like `lambdef` and `lambdef_nocond` have been merged in the + syntax tree to make it easier to do deal with them. + +Parser Tree Classes +------------------- +""" + +import re + +from parso._compatibility import utf8_repr, unicode +from parso.tree import Node, BaseNode, Leaf, ErrorNode, ErrorLeaf, \ + search_ancestor +from parso.python.prefix import split_prefix + +_FLOW_CONTAINERS = set(['if_stmt', 'while_stmt', 'for_stmt', 'try_stmt', + 'with_stmt', 'async_stmt', 'suite']) +_RETURN_STMT_CONTAINERS = set(['suite', 'simple_stmt']) | _FLOW_CONTAINERS +_FUNC_CONTAINERS = set(['suite', 'simple_stmt', 'decorated']) | _FLOW_CONTAINERS +_GET_DEFINITION_TYPES = set([ + 'expr_stmt', 'comp_for', 'with_stmt', 'for_stmt', 'import_name', + 'import_from', 'param' +]) +_IMPORTS = set(['import_name', 'import_from']) + + + +class DocstringMixin(object): + __slots__ = () + + def get_doc_node(self): + """ + Returns the string leaf of a docstring. e.g. ``r'''foo'''``. + """ + if self.type == 'file_input': + node = self.children[0] + elif self.type in ('funcdef', 'classdef'): + node = self.children[self.children.index(':') + 1] + if node.type == 'suite': # Normally a suite + node = node.children[1] # -> NEWLINE stmt + else: # ExprStmt + simple_stmt = self.parent + c = simple_stmt.parent.children + index = c.index(simple_stmt) + if not index: + return None + node = c[index - 1] + + if node.type == 'simple_stmt': + node = node.children[0] + if node.type == 'string': + return node + return None + + +class PythonMixin(object): + """ + Some Python specific utitilies. + """ + __slots__ = () + + def get_name_of_position(self, position): + """ + Given a (line, column) tuple, returns a :py:class:`Name` or ``None`` if + there is no name at that position. + """ + for c in self.children: + if isinstance(c, Leaf): + if c.type == 'name' and c.start_pos <= position <= c.end_pos: + return c + else: + result = c.get_name_of_position(position) + if result is not None: + return result + return None + + +class PythonLeaf(PythonMixin, Leaf): + __slots__ = () + + def _split_prefix(self): + return split_prefix(self, self.get_start_pos_of_prefix()) + + def get_start_pos_of_prefix(self): + """ + Basically calls :py:meth:`parso.tree.NodeOrLeaf.get_start_pos_of_prefix`. + """ + # TODO it is really ugly that we have to override it. Maybe change + # indent error leafs somehow? No idea how, though. + previous_leaf = self.get_previous_leaf() + if previous_leaf is not None and previous_leaf.type == 'error_leaf' \ + and previous_leaf.original_type in ('indent', 'error_dedent'): + previous_leaf = previous_leaf.get_previous_leaf() + + if previous_leaf is None: + return self.line - self.prefix.count('\n'), 0 # It's the first leaf. + return previous_leaf.end_pos + + + +class _LeafWithoutNewlines(PythonLeaf): + """ + Simply here to optimize performance. + """ + __slots__ = () + + @property + def end_pos(self): + return self.line, self.column + len(self.value) + + +# Python base classes +class PythonBaseNode(PythonMixin, BaseNode): + __slots__ = () + + +class PythonNode(PythonMixin, Node): + __slots__ = () + + +class PythonErrorNode(PythonMixin, ErrorNode): + __slots__ = () + + +class PythonErrorLeaf(ErrorLeaf, PythonLeaf): + __slots__ = () + + +class EndMarker(_LeafWithoutNewlines): + __slots__ = () + type = 'endmarker' + + +class Newline(PythonLeaf): + """Contains NEWLINE and ENDMARKER tokens.""" + __slots__ = () + type = 'newline' + + @utf8_repr + def __repr__(self): + return "<%s: %s>" % (type(self).__name__, repr(self.value)) + + +class Name(_LeafWithoutNewlines): + """ + A string. Sometimes it is important to know if the string belongs to a name + or not. + """ + type = 'name' + __slots__ = () + + def __repr__(self): + return "<%s: %s@%s,%s>" % (type(self).__name__, self.value, + self.line, self.column) + + def is_definition(self): + """ + Returns True if the name is being defined. + """ + return self.get_definition() is not None + + def get_definition(self, import_name_always=False): + """ + Returns None if there's on definition for a name. + + :param import_name_alway: Specifies if an import name is always a + definition. Normally foo in `from foo import bar` is not a + definition. + """ + node = self.parent + type_ = node.type + if type_ in ('power', 'atom_expr'): + # In `self.x = 3` self is not a definition, but x is. + return None + + if type_ in ('funcdef', 'classdef'): + if self == node.name: + return node + return None + + if type_ == 'except_clause': + # TODO in Python 2 this doesn't work correctly. See grammar file. + # I think we'll just let it be. Python 2 will be gone in a few + # years. + if self.get_previous_sibling() == 'as': + return node.parent # The try_stmt. + return None + + while node is not None: + if node.type == 'suite': + return None + if node.type in _GET_DEFINITION_TYPES: + if self in node.get_defined_names(): + return node + if import_name_always and node.type in _IMPORTS: + return node + return None + node = node.parent + return None + + + +class Literal(PythonLeaf): + __slots__ = () + + +class Number(Literal): + type = 'number' + __slots__ = () + + +class String(Literal): + type = 'string' + __slots__ = () + + @property + def string_prefix(self): + return re.match('\w*(?=[\'"])', self.value).group(0) + + def _get_payload(self): + match = re.search( + r'''('{3}|"{3}|'|")(.*)$''', + self.value, + flags=re.DOTALL + ) + return match.group(2)[:-len(match.group(1))] + + +class _StringComparisonMixin(object): + def __eq__(self, other): + """ + Make comparisons with strings easy. + Improves the readability of the parser. + """ + if isinstance(other, (str, unicode)): + return self.value == other + + return self is other + + def __ne__(self, other): + """Python 2 compatibility.""" + return not self.__eq__(other) + + def __hash__(self): + return hash(self.value) + + +class Operator(_LeafWithoutNewlines, _StringComparisonMixin): + type = 'operator' + __slots__ = () + + +class Keyword(_LeafWithoutNewlines, _StringComparisonMixin): + type = 'keyword' + __slots__ = () + + +class Scope(PythonBaseNode, DocstringMixin): + """ + Super class for the parser tree, which represents the state of a python + text file. + A Scope is either a function, class or lambda. + """ + __slots__ = () + + def __init__(self, children): + super(Scope, self).__init__(children) + + def iter_funcdefs(self): + """ + Returns a generator of `funcdef` nodes. + """ + return self._search_in_scope('funcdef') + + def iter_classdefs(self): + """ + Returns a generator of `classdef` nodes. + """ + return self._search_in_scope('classdef') + + def iter_imports(self): + """ + Returns a generator of `import_name` and `import_from` nodes. + """ + return self._search_in_scope('import_name', 'import_from') + + def _search_in_scope(self, *names): + def scan(children): + for element in children: + if element.type in names: + yield element + if element.type in _FUNC_CONTAINERS: + for e in scan(element.children): + yield e + + return scan(self.children) + + def get_suite(self): + """ + Returns the part that is executed by the function. + """ + return self.children[-1] + + def __repr__(self): + try: + name = self.name.value + except AttributeError: + name = '' + + return "<%s: %s@%s-%s>" % (type(self).__name__, name, + self.start_pos[0], self.end_pos[0]) + + +class Module(Scope): + """ + The top scope, which is always a module. + Depending on the underlying parser this may be a full module or just a part + of a module. + """ + __slots__ = ('_used_names',) + type = 'file_input' + + def __init__(self, children): + super(Module, self).__init__(children) + self._used_names = None + + def _iter_future_import_names(self): + """ + :return: A list of future import names. + :rtype: list of str + """ + # In Python it's not allowed to use future imports after the first + # actual (non-future) statement. However this is not a linter here, + # just return all future imports. If people want to scan for issues + # they should use the API. + for imp in self.iter_imports(): + if imp.type == 'import_from' and imp.level == 0: + for path in imp.get_paths(): + names = [name.value for name in path] + if len(names) == 2 and names[0] == '__future__': + yield names[1] + + def _has_explicit_absolute_import(self): + """ + Checks if imports in this module are explicitly absolute, i.e. there + is a ``__future__`` import. + Currently not public, might be in the future. + :return bool: + """ + for name in self._iter_future_import_names(): + if name == 'absolute_import': + return True + return False + + def get_used_names(self): + """ + Returns all the :class:`Name` leafs that exist in this module. This + includes both definitions and references of names. + """ + if self._used_names is None: + # Don't directly use self._used_names to eliminate a lookup. + dct = {} + + def recurse(node): + try: + children = node.children + except AttributeError: + if node.type == 'name': + arr = dct.setdefault(node.value, []) + arr.append(node) + else: + for child in children: + recurse(child) + + recurse(self) + self._used_names = dct + return self._used_names + + +class Decorator(PythonBaseNode): + type = 'decorator' + __slots__ = () + + +class ClassOrFunc(Scope): + __slots__ = () + + @property + def name(self): + """ + Returns the `Name` leaf that defines the function or class name. + """ + return self.children[1] + + def get_decorators(self): + """ + :rtype: list of :class:`Decorator` + """ + decorated = self.parent + if decorated.type == 'decorated': + if decorated.children[0].type == 'decorators': + return decorated.children[0].children + else: + return decorated.children[:1] + else: + return [] + + +class Class(ClassOrFunc): + """ + Used to store the parsed contents of a python class. + """ + type = 'classdef' + __slots__ = () + + def __init__(self, children): + super(Class, self).__init__(children) + + def get_super_arglist(self): + """ + Returns the `arglist` node that defines the super classes. It returns + None if there are no arguments. + """ + if self.children[2] != '(': # Has no parentheses + return None + else: + if self.children[3] == ')': # Empty parentheses + return None + else: + return self.children[3] + + +def _create_params(parent, argslist_list): + """ + `argslist_list` is a list that can contain an argslist as a first item, but + most not. It's basically the items between the parameter brackets (which is + at most one item). + This function modifies the parser structure. It generates `Param` objects + from the normal ast. Those param objects do not exist in a normal ast, but + make the evaluation of the ast tree so much easier. + You could also say that this function replaces the argslist node with a + list of Param objects. + """ + def check_python2_nested_param(node): + """ + Python 2 allows params to look like ``def x(a, (b, c))``, which is + basically a way of unpacking tuples in params. Python 3 has ditched + this behavior. Jedi currently just ignores those constructs. + """ + return node.type == 'fpdef' and node.children[0] == '(' + + try: + first = argslist_list[0] + except IndexError: + return [] + + if first.type in ('name', 'fpdef'): + if check_python2_nested_param(first): + return [first] + else: + return [Param([first], parent)] + elif first == '*': + return [first] + else: # argslist is a `typedargslist` or a `varargslist`. + if first.type == 'tfpdef': + children = [first] + else: + children = first.children + new_children = [] + start = 0 + # Start with offset 1, because the end is higher. + for end, child in enumerate(children + [None], 1): + if child is None or child == ',': + param_children = children[start:end] + if param_children: # Could as well be comma and then end. + if param_children[0] == '*' and param_children[1] == ',' \ + or check_python2_nested_param(param_children[0]): + for p in param_children: + p.parent = parent + new_children += param_children + else: + new_children.append(Param(param_children, parent)) + start = end + return new_children + + +class Function(ClassOrFunc): + """ + Used to store the parsed contents of a python function. + + Children:: + + 0. + 1. + 2. parameter list (including open-paren and close-paren s) + 3. or 5. + 4. or 6. Node() representing function body + 3. -> (if annotation is also present) + 4. annotation (if present) + """ + type = 'funcdef' + + def __init__(self, children): + super(Function, self).__init__(children) + parameters = self.children[2] # After `def foo` + parameters.children[1:-1] = _create_params(parameters, parameters.children[1:-1]) + + def _get_param_nodes(self): + return self.children[2].children + + def get_params(self): + """ + Returns a list of `Param()`. + """ + return [p for p in self._get_param_nodes() if p.type == 'param'] + + @property + def name(self): + return self.children[1] # First token after `def` + + def iter_yield_exprs(self): + """ + Returns a generator of `yield_expr`. + """ + def scan(children): + for element in children: + if element.type in ('classdef', 'funcdef', 'lambdef'): + continue + + try: + nested_children = element.children + except AttributeError: + if element.value == 'yield': + if element.parent.type == 'yield_expr': + yield element.parent + else: + yield element + else: + for result in scan(nested_children): + yield result + + return scan(self.children) + + def iter_return_stmts(self): + """ + Returns a generator of `return_stmt`. + """ + def scan(children): + for element in children: + if element.type == 'return_stmt' \ + or element.type == 'keyword' and element.value == 'return': + yield element + if element.type in _RETURN_STMT_CONTAINERS: + for e in scan(element.children): + yield e + + return scan(self.children) + + def iter_raise_stmts(self): + """ + Returns a generator of `raise_stmt`. Includes raise statements inside try-except blocks + """ + def scan(children): + for element in children: + if element.type == 'raise_stmt' \ + or element.type == 'keyword' and element.value == 'raise': + yield element + if element.type in _RETURN_STMT_CONTAINERS: + for e in scan(element.children): + yield e + + return scan(self.children) + + def is_generator(self): + """ + :return bool: Checks if a function is a generator or not. + """ + return next(self.iter_yield_exprs(), None) is not None + + @property + def annotation(self): + """ + Returns the test node after `->` or `None` if there is no annotation. + """ + try: + if self.children[3] == "->": + return self.children[4] + assert self.children[3] == ":" + return None + except IndexError: + return None + +class Lambda(Function): + """ + Lambdas are basically trimmed functions, so give it the same interface. + + Children:: + + 0. + *. for each argument x + -2. + -1. Node() representing body + """ + type = 'lambdef' + __slots__ = () + + def __init__(self, children): + # We don't want to call the Function constructor, call its parent. + super(Function, self).__init__(children) + # Everything between `lambda` and the `:` operator is a parameter. + self.children[1:-2] = _create_params(self, self.children[1:-2]) + + @property + def name(self): + """ + Raises an AttributeError. Lambdas don't have a defined name. + """ + raise AttributeError("lambda is not named.") + + def _get_param_nodes(self): + return self.children[1:-2] + + @property + def annotation(self): + """ + Returns `None`, lambdas don't have annotations. + """ + return None + + def __repr__(self): + return "<%s@%s>" % (self.__class__.__name__, self.start_pos) + + +class Flow(PythonBaseNode): + __slots__ = () + + +class IfStmt(Flow): + type = 'if_stmt' + __slots__ = () + + def get_test_nodes(self): + """ + E.g. returns all the `test` nodes that are named as x, below: + + if x: + pass + elif x: + pass + """ + for i, c in enumerate(self.children): + if c in ('elif', 'if'): + yield self.children[i + 1] + + def get_corresponding_test_node(self, node): + """ + Searches for the branch in which the node is and returns the + corresponding test node (see function above). However if the node is in + the test node itself and not in the suite return None. + """ + start_pos = node.start_pos + for check_node in reversed(list(self.get_test_nodes())): + if check_node.start_pos < start_pos: + if start_pos < check_node.end_pos: + return None + # In this case the node is within the check_node itself, + # not in the suite + else: + return check_node + + def is_node_after_else(self, node): + """ + Checks if a node is defined after `else`. + """ + for c in self.children: + if c == 'else': + if node.start_pos > c.start_pos: + return True + else: + return False + + +class WhileStmt(Flow): + type = 'while_stmt' + __slots__ = () + + +class ForStmt(Flow): + type = 'for_stmt' + __slots__ = () + + def get_testlist(self): + """ + Returns the input node ``y`` from: ``for x in y:``. + """ + return self.children[3] + + def get_defined_names(self): + return _defined_names(self.children[1]) + + +class TryStmt(Flow): + type = 'try_stmt' + __slots__ = () + + def get_except_clause_tests(self): + """ + Returns the ``test`` nodes found in ``except_clause`` nodes. + Returns ``[None]`` for except clauses without an exception given. + """ + for node in self.children: + if node.type == 'except_clause': + yield node.children[1] + elif node == 'except': + yield None + + +class WithStmt(Flow): + type = 'with_stmt' + __slots__ = () + + def get_defined_names(self): + """ + Returns the a list of `Name` that the with statement defines. The + defined names are set after `as`. + """ + names = [] + for with_item in self.children[1:-2:2]: + # Check with items for 'as' names. + if with_item.type == 'with_item': + names += _defined_names(with_item.children[2]) + return names + + def get_test_node_from_name(self, name): + node = name.parent + if node.type != 'with_item': + raise ValueError('The name is not actually part of a with statement.') + return node.children[0] + + +class Import(PythonBaseNode): + __slots__ = () + + def get_path_for_name(self, name): + """ + The path is the list of names that leads to the searched name. + + :return list of Name: + """ + try: + # The name may be an alias. If it is, just map it back to the name. + name = self._aliases()[name] + except KeyError: + pass + + for path in self.get_paths(): + if name in path: + return path[:path.index(name) + 1] + raise ValueError('Name should be defined in the import itself') + + def is_nested(self): + return False # By default, sub classes may overwrite this behavior + + def is_star_import(self): + return self.children[-1] == '*' + + +class ImportFrom(Import): + type = 'import_from' + __slots__ = () + + def get_defined_names(self): + """ + Returns the a list of `Name` that the import defines. The + defined names are set after `import` or in case an alias - `as` - is + present that name is returned. + """ + return [alias or name for name, alias in self._as_name_tuples()] + + def _aliases(self): + """Mapping from alias to its corresponding name.""" + return dict((alias, name) for name, alias in self._as_name_tuples() + if alias is not None) + + def get_from_names(self): + for n in self.children[1:]: + if n not in ('.', '...'): + break + if n.type == 'dotted_name': # from x.y import + return n.children[::2] + elif n == 'import': # from . import + return [] + else: # from x import + return [n] + + @property + def level(self): + """The level parameter of ``__import__``.""" + level = 0 + for n in self.children[1:]: + if n in ('.', '...'): + level += len(n.value) + else: + break + return level + + def _as_name_tuples(self): + last = self.children[-1] + if last == ')': + last = self.children[-2] + elif last == '*': + return # No names defined directly. + + if last.type == 'import_as_names': + as_names = last.children[::2] + else: + as_names = [last] + for as_name in as_names: + if as_name.type == 'name': + yield as_name, None + else: + yield as_name.children[::2] # yields x, y -> ``x as y`` + + def get_paths(self): + """ + The import paths defined in an import statement. Typically an array + like this: ``[, ]``. + + :return list of list of Name: + """ + dotted = self.get_from_names() + + if self.children[-1] == '*': + return [dotted] + return [dotted + [name] for name, alias in self._as_name_tuples()] + + +class ImportName(Import): + """For ``import_name`` nodes. Covers normal imports without ``from``.""" + type = 'import_name' + __slots__ = () + + def get_defined_names(self): + """ + Returns the a list of `Name` that the import defines. The defined names + is always the first name after `import` or in case an alias - `as` - is + present that name is returned. + """ + return [alias or path[0] for path, alias in self._dotted_as_names()] + + @property + def level(self): + """The level parameter of ``__import__``.""" + return 0 # Obviously 0 for imports without from. + + def get_paths(self): + return [path for path, alias in self._dotted_as_names()] + + def _dotted_as_names(self): + """Generator of (list(path), alias) where alias may be None.""" + dotted_as_names = self.children[1] + if dotted_as_names.type == 'dotted_as_names': + as_names = dotted_as_names.children[::2] + else: + as_names = [dotted_as_names] + + for as_name in as_names: + if as_name.type == 'dotted_as_name': + alias = as_name.children[2] + as_name = as_name.children[0] + else: + alias = None + if as_name.type == 'name': + yield [as_name], alias + else: + # dotted_names + yield as_name.children[::2], alias + + def is_nested(self): + """ + This checks for the special case of nested imports, without aliases and + from statement:: + + import foo.bar + """ + return bool([1 for path, alias in self._dotted_as_names() + if alias is None and len(path) > 1]) + + def _aliases(self): + """ + :return list of Name: Returns all the alias + """ + return dict((alias, path[-1]) for path, alias in self._dotted_as_names() + if alias is not None) + + +class KeywordStatement(PythonBaseNode): + """ + For the following statements: `assert`, `del`, `global`, `nonlocal`, + `raise`, `return`, `yield`, `return`, `yield`. + + `pass`, `continue` and `break` are not in there, because they are just + simple keywords and the parser reduces it to a keyword. + """ + __slots__ = () + + @property + def type(self): + """ + Keyword statements start with the keyword and end with `_stmt`. You can + crosscheck this with the Python grammar. + """ + return '%s_stmt' % self.keyword + + @property + def keyword(self): + return self.children[0].value + + +class AssertStmt(KeywordStatement): + __slots__ = () + + @property + def assertion(self): + return self.children[1] + + +class GlobalStmt(KeywordStatement): + __slots__ = () + + def get_global_names(self): + return self.children[1::2] + + +class ReturnStmt(KeywordStatement): + __slots__ = () + + +class YieldExpr(PythonBaseNode): + type = 'yield_expr' + __slots__ = () + + +def _defined_names(current): + """ + A helper function to find the defined names in statements, for loops and + list comprehensions. + """ + names = [] + if current.type in ('testlist_star_expr', 'testlist_comp', 'exprlist', 'testlist'): + for child in current.children[::2]: + names += _defined_names(child) + elif current.type in ('atom', 'star_expr'): + names += _defined_names(current.children[1]) + elif current.type in ('power', 'atom_expr'): + if current.children[-2] != '**': # Just if there's no operation + trailer = current.children[-1] + if trailer.children[0] == '.': + names.append(trailer.children[1]) + else: + names.append(current) + return names + + +class ExprStmt(PythonBaseNode, DocstringMixin): + type = 'expr_stmt' + __slots__ = () + + def get_defined_names(self): + """ + Returns a list of `Name` defined before the `=` sign. + """ + names = [] + if self.children[1].type == 'annassign': + names = _defined_names(self.children[0]) + return [ + name + for i in range(0, len(self.children) - 2, 2) + if '=' in self.children[i + 1].value + for name in _defined_names(self.children[i]) + ] + names + + def get_rhs(self): + """Returns the right-hand-side of the equals.""" + return self.children[-1] + + def yield_operators(self): + """ + Returns a generator of `+=`, `=`, etc. or None if there is no operation. + """ + first = self.children[1] + if first.type == 'annassign': + if len(first.children) <= 2: + return # No operator is available, it's just PEP 484. + + first = first.children[2] + yield first + + for operator in self.children[3::2]: + yield operator + + +class Param(PythonBaseNode): + """ + It's a helper class that makes business logic with params much easier. The + Python grammar defines no ``param`` node. It defines it in a different way + that is not really suited to working with parameters. + """ + type = 'param' + + def __init__(self, children, parent): + super(Param, self).__init__(children) + self.parent = parent + for child in children: + child.parent = self + + @property + def star_count(self): + """ + Is `0` in case of `foo`, `1` in case of `*foo` or `2` in case of + `**foo`. + """ + first = self.children[0] + if first in ('*', '**'): + return len(first.value) + return 0 + + @property + def default(self): + """ + The default is the test node that appears after the `=`. Is `None` in + case no default is present. + """ + has_comma = self.children[-1] == ',' + try: + if self.children[-2 - int(has_comma)] == '=': + return self.children[-1 - int(has_comma)] + except IndexError: + return None + + @property + def annotation(self): + """ + The default is the test node that appears after `:`. Is `None` in case + no annotation is present. + """ + tfpdef = self._tfpdef() + if tfpdef.type == 'tfpdef': + assert tfpdef.children[1] == ":" + assert len(tfpdef.children) == 3 + annotation = tfpdef.children[2] + return annotation + else: + return None + + def _tfpdef(self): + """ + tfpdef: see e.g. grammar36.txt. + """ + offset = int(self.children[0] in ('*', '**')) + return self.children[offset] + + @property + def name(self): + """ + The `Name` leaf of the param. + """ + if self._tfpdef().type == 'tfpdef': + return self._tfpdef().children[0] + else: + return self._tfpdef() + + def get_defined_names(self): + return [self.name] + + @property + def position_index(self): + """ + Property for the positional index of a paramter. + """ + index = self.parent.children.index(self) + try: + keyword_only_index = self.parent.children.index('*') + if index > keyword_only_index: + # Skip the ` *, ` + index -= 2 + except ValueError: + pass + return index - 1 + + def get_parent_function(self): + """ + Returns the function/lambda of a parameter. + """ + return search_ancestor(self, 'funcdef', 'lambdef') + + def get_code(self, include_prefix=True, include_comma=True): + """ + Like all the other get_code functions, but includes the param + `include_comma`. + + :param include_comma bool: If enabled includes the comma in the string output. + """ + if include_comma: + return super(Param, self).get_code(include_prefix) + + children = self.children + if children[-1] == ',': + children = children[:-1] + return self._get_code_for_children( + children, + include_prefix=include_prefix + ) + + def __repr__(self): + default = '' if self.default is None else '=%s' % self.default.get_code() + return '<%s: %s>' % (type(self).__name__, str(self._tfpdef()) + default) + + +class CompFor(PythonBaseNode): + type = 'comp_for' + __slots__ = () + + def get_defined_names(self): + """ + Returns the a list of `Name` that the comprehension defines. + """ + # allow async for + return _defined_names(self.children[self.children.index('for') + 1]) diff --git a/pythonFiles/parso/tree.py b/pythonFiles/parso/tree.py new file mode 100644 index 000000000000..72a14945b0f6 --- /dev/null +++ b/pythonFiles/parso/tree.py @@ -0,0 +1,364 @@ +from abc import abstractmethod, abstractproperty +from parso._compatibility import utf8_repr, encoding, py_version + + +def search_ancestor(node, *node_types): + """ + Recursively looks at the parents of a node and returns the first found node + that matches node_types. Returns ``None`` if no matching node is found. + + :param node: The ancestors of this node will be checked. + :param node_types: type names that are searched for. + :type node_types: tuple of str + """ + while True: + node = node.parent + if node is None or node.type in node_types: + return node + + +class NodeOrLeaf(object): + """ + The base class for nodes and leaves. + """ + __slots__ = () + type = None + ''' + The type is a string that typically matches the types of the grammar file. + ''' + + def get_root_node(self): + """ + Returns the root node of a parser tree. The returned node doesn't have + a parent node like all the other nodes/leaves. + """ + scope = self + while scope.parent is not None: + scope = scope.parent + return scope + + def get_next_sibling(self): + """ + Returns the node immediately following this node in this parent's + children list. If this node does not have a next sibling, it is None + """ + # Can't use index(); we need to test by identity + for i, child in enumerate(self.parent.children): + if child is self: + try: + return self.parent.children[i + 1] + except IndexError: + return None + + def get_previous_sibling(self): + """ + Returns the node immediately preceding this node in this parent's + children list. If this node does not have a previous sibling, it is + None. + None. + """ + # Can't use index(); we need to test by identity + for i, child in enumerate(self.parent.children): + if child is self: + if i == 0: + return None + return self.parent.children[i - 1] + + def get_previous_leaf(self): + """ + Returns the previous leaf in the parser tree. + Returns `None` if this is the first element in the parser tree. + """ + node = self + while True: + c = node.parent.children + i = c.index(node) + if i == 0: + node = node.parent + if node.parent is None: + return None + else: + node = c[i - 1] + break + + while True: + try: + node = node.children[-1] + except AttributeError: # A Leaf doesn't have children. + return node + + def get_next_leaf(self): + """ + Returns the next leaf in the parser tree. + Returns None if this is the last element in the parser tree. + """ + node = self + while True: + c = node.parent.children + i = c.index(node) + if i == len(c) - 1: + node = node.parent + if node.parent is None: + return None + else: + node = c[i + 1] + break + + while True: + try: + node = node.children[0] + except AttributeError: # A Leaf doesn't have children. + return node + + @abstractproperty + def start_pos(self): + """ + Returns the starting position of the prefix as a tuple, e.g. `(3, 4)`. + + :return tuple of int: (line, column) + """ + + @abstractproperty + def end_pos(self): + """ + Returns the end position of the prefix as a tuple, e.g. `(3, 4)`. + + :return tuple of int: (line, column) + """ + + @abstractmethod + def get_start_pos_of_prefix(self): + """ + Returns the start_pos of the prefix. This means basically it returns + the end_pos of the last prefix. The `get_start_pos_of_prefix()` of the + prefix `+` in `2 + 1` would be `(1, 1)`, while the start_pos is + `(1, 2)`. + + :return tuple of int: (line, column) + """ + + @abstractmethod + def get_first_leaf(self): + """ + Returns the first leaf of a node or itself if this is a leaf. + """ + + @abstractmethod + def get_last_leaf(self): + """ + Returns the last leaf of a node or itself if this is a leaf. + """ + + @abstractmethod + def get_code(self, include_prefix=True): + """ + Returns the code that was input the input for the parser for this node. + + :param include_prefix: Removes the prefix (whitespace and comments) of + e.g. a statement. + """ + + +class Leaf(NodeOrLeaf): + ''' + Leafs are basically tokens with a better API. Leafs exactly know where they + were defined and what text preceeds them. + ''' + __slots__ = ('value', 'parent', 'line', 'column', 'prefix') + + def __init__(self, value, start_pos, prefix=''): + self.value = value + ''' + :py:func:`str` The value of the current token. + ''' + self.start_pos = start_pos + self.prefix = prefix + ''' + :py:func:`str` Typically a mixture of whitespace and comments. Stuff + that is syntactically irrelevant for the syntax tree. + ''' + self.parent = None + ''' + The parent :class:`BaseNode` of this leaf. + ''' + + @property + def start_pos(self): + return self.line, self.column + + @start_pos.setter + def start_pos(self, value): + self.line = value[0] + self.column = value[1] + + def get_start_pos_of_prefix(self): + previous_leaf = self.get_previous_leaf() + if previous_leaf is None: + return self.line - self.prefix.count('\n'), 0 # It's the first leaf. + return previous_leaf.end_pos + + def get_first_leaf(self): + return self + + def get_last_leaf(self): + return self + + def get_code(self, include_prefix=True): + if include_prefix: + return self.prefix + self.value + else: + return self.value + + @property + def end_pos(self): + lines = self.value.split('\n') + end_pos_line = self.line + len(lines) - 1 + # Check for multiline token + if self.line == end_pos_line: + end_pos_column = self.column + len(lines[-1]) + else: + end_pos_column = len(lines[-1]) + return end_pos_line, end_pos_column + + @utf8_repr + def __repr__(self): + value = self.value + if not value: + value = self.type + return "<%s: %s>" % (type(self).__name__, value) + + +class TypedLeaf(Leaf): + __slots__ = ('type',) + def __init__(self, type, value, start_pos, prefix=''): + super(TypedLeaf, self).__init__(value, start_pos, prefix) + self.type = type + + +class BaseNode(NodeOrLeaf): + """ + The super class for all nodes. + A node has children, a type and possibly a parent node. + """ + __slots__ = ('children', 'parent') + type = None + + def __init__(self, children): + for c in children: + c.parent = self + self.children = children + """ + A list of :class:`NodeOrLeaf` child nodes. + """ + self.parent = None + ''' + The parent :class:`BaseNode` of this leaf. + None if this is the root node. + ''' + + @property + def start_pos(self): + return self.children[0].start_pos + + def get_start_pos_of_prefix(self): + return self.children[0].get_start_pos_of_prefix() + + @property + def end_pos(self): + return self.children[-1].end_pos + + def _get_code_for_children(self, children, include_prefix): + if include_prefix: + return "".join(c.get_code() for c in children) + else: + first = children[0].get_code(include_prefix=False) + return first + "".join(c.get_code() for c in children[1:]) + + def get_code(self, include_prefix=True): + return self._get_code_for_children(self.children, include_prefix) + + def get_leaf_for_position(self, position, include_prefixes=False): + """ + Get the :py:class:`parso.tree.Leaf` at ``position`` + + :param tuple position: A position tuple, row, column. Rows start from 1 + :param bool include_prefixes: If ``False``, ``None`` will be returned if ``position`` falls + on whitespace or comments before a leaf + :return: :py:class:`parso.tree.Leaf` at ``position``, or ``None`` + """ + def binary_search(lower, upper): + if lower == upper: + element = self.children[lower] + if not include_prefixes and position < element.start_pos: + # We're on a prefix. + return None + # In case we have prefixes, a leaf always matches + try: + return element.get_leaf_for_position(position, include_prefixes) + except AttributeError: + return element + + + index = int((lower + upper) / 2) + element = self.children[index] + if position <= element.end_pos: + return binary_search(lower, index) + else: + return binary_search(index + 1, upper) + + if not ((1, 0) <= position <= self.children[-1].end_pos): + raise ValueError('Please provide a position that exists within this node.') + return binary_search(0, len(self.children) - 1) + + def get_first_leaf(self): + return self.children[0].get_first_leaf() + + def get_last_leaf(self): + return self.children[-1].get_last_leaf() + + @utf8_repr + def __repr__(self): + code = self.get_code().replace('\n', ' ').strip() + if not py_version >= 30: + code = code.encode(encoding, 'replace') + return "<%s: %s@%s,%s>" % \ + (type(self).__name__, code, self.start_pos[0], self.start_pos[1]) + + +class Node(BaseNode): + """Concrete implementation for interior nodes.""" + __slots__ = ('type',) + + def __init__(self, type, children): + super(Node, self).__init__(children) + self.type = type + + def __repr__(self): + return "%s(%s, %r)" % (self.__class__.__name__, self.type, self.children) + + +class ErrorNode(BaseNode): + """ + A node that containes valid nodes/leaves that we're follow by a token that + was invalid. This basically means that the leaf after this node is where + Python would mark a syntax error. + """ + __slots__ = () + type = 'error_node' + + +class ErrorLeaf(Leaf): + """ + A leaf that is either completely invalid in a language (like `$` in Python) + or is invalid at that position. Like the star in `1 +* 1`. + """ + __slots__ = ('original_type',) + type = 'error_leaf' + + def __init__(self, original_type, value, start_pos, prefix=''): + super(ErrorLeaf, self).__init__(value, start_pos, prefix) + self.original_type = original_type + + def __repr__(self): + return "<%s: %s:%s, %s>" % \ + (type(self).__name__, self.original_type, repr(self.value), self.start_pos) diff --git a/pythonFiles/parso/utils.py b/pythonFiles/parso/utils.py new file mode 100644 index 000000000000..a4801b996616 --- /dev/null +++ b/pythonFiles/parso/utils.py @@ -0,0 +1,156 @@ +from collections import namedtuple +import re +import sys +from ast import literal_eval + +from parso._compatibility import unicode, total_ordering + + +Version = namedtuple('Version', 'major, minor, micro') + + +def split_lines(string, keepends=False): + r""" + Intended for Python code. In contrast to Python's :py:meth:`str.splitlines`, + looks at form feeds and other special characters as normal text. Just + splits ``\n`` and ``\r\n``. + Also different: Returns ``[""]`` for an empty string input. + + In Python 2.7 form feeds are used as normal characters when using + str.splitlines. However in Python 3 somewhere there was a decision to split + also on form feeds. + """ + if keepends: + lst = string.splitlines(True) + + # We have to merge lines that were broken by form feed characters. + merge = [] + for i, line in enumerate(lst): + if line.endswith('\f'): + merge.append(i) + + for index in reversed(merge): + try: + lst[index] = lst[index] + lst[index + 1] + del lst[index + 1] + except IndexError: + # index + 1 can be empty and therefore there's no need to + # merge. + pass + + # The stdlib's implementation of the end is inconsistent when calling + # it with/without keepends. One time there's an empty string in the + # end, one time there's none. + if string.endswith('\n') or string == '': + lst.append('') + return lst + else: + return re.split('\n|\r\n', string) + + +def python_bytes_to_unicode(source, encoding='utf-8', errors='strict'): + """ + Checks for unicode BOMs and PEP 263 encoding declarations. Then returns a + unicode object like in :py:meth:`bytes.decode`. + + :param encoding: See :py:meth:`bytes.decode` documentation. + :param errors: See :py:meth:`bytes.decode` documentation. ``errors`` can be + ``'strict'``, ``'replace'`` or ``'ignore'``. + """ + def detect_encoding(): + """ + For the implementation of encoding definitions in Python, look at: + - http://www.python.org/dev/peps/pep-0263/ + - http://docs.python.org/2/reference/lexical_analysis.html#encoding-declarations + """ + byte_mark = literal_eval(r"b'\xef\xbb\xbf'") + if source.startswith(byte_mark): + # UTF-8 byte-order mark + return 'utf-8' + + first_two_lines = re.match(br'(?:[^\n]*\n){0,2}', source).group(0) + possible_encoding = re.search(br"coding[=:]\s*([-\w.]+)", + first_two_lines) + if possible_encoding: + return possible_encoding.group(1) + else: + # the default if nothing else has been set -> PEP 263 + return encoding + + if isinstance(source, unicode): + # only cast str/bytes + return source + + encoding = detect_encoding() + if not isinstance(encoding, unicode): + encoding = unicode(encoding, 'utf-8', 'replace') + + # Cast to unicode + return unicode(source, encoding, errors) + + +def version_info(): + """ + Returns a namedtuple of parso's version, similar to Python's + ``sys.version_info``. + """ + from parso import __version__ + tupl = re.findall(r'[a-z]+|\d+', __version__) + return Version(*[x if i == 3 else int(x) for i, x in enumerate(tupl)]) + + +def _parse_version(version): + match = re.match(r'(\d+)(?:\.(\d)(?:\.\d+)?)?$', version) + if match is None: + raise ValueError('The given version is not in the right format. ' + 'Use something like "3.2" or "3".') + + major = int(match.group(1)) + minor = match.group(2) + if minor is None: + # Use the latest Python in case it's not exactly defined, because the + # grammars are typically backwards compatible? + if major == 2: + minor = "7" + elif major == 3: + minor = "6" + else: + raise NotImplementedError("Sorry, no support yet for those fancy new/old versions.") + minor = int(minor) + return PythonVersionInfo(major, minor) + + +@total_ordering +class PythonVersionInfo(namedtuple('Version', 'major, minor')): + def __gt__(self, other): + if isinstance(other, tuple): + if len(other) != 2: + raise ValueError("Can only compare to tuples of length 2.") + return (self.major, self.minor) > other + super(PythonVersionInfo, self).__gt__(other) + + return (self.major, self.minor) + + def __eq__(self, other): + if isinstance(other, tuple): + if len(other) != 2: + raise ValueError("Can only compare to tuples of length 2.") + return (self.major, self.minor) == other + super(PythonVersionInfo, self).__eq__(other) + + def __ne__(self, other): + return not self.__eq__(other) + + +def parse_version_string(version=None): + """ + Checks for a valid version number (e.g. `3.2` or `2.7.1` or `3`) and + returns a corresponding version info that is always two characters long in + decimal. + """ + if version is None: + version = '%s.%s' % sys.version_info[:2] + if not isinstance(version, (unicode, str)): + raise TypeError("version must be a string like 3.2.") + + return _parse_version(version) diff --git a/pythonFiles/preview/jedi/__init__.py b/pythonFiles/preview/jedi/__init__.py deleted file mode 100644 index c2782760c0b0..000000000000 --- a/pythonFiles/preview/jedi/__init__.py +++ /dev/null @@ -1,43 +0,0 @@ -""" -Jedi is a static analysis tool for Python that can be used in IDEs/editors. Its -historic focus is autocompletion, but does static analysis for now as well. -Jedi is fast and is very well tested. It understands Python on a deeper level -than all other static analysis frameworks for Python. - -Jedi has support for two different goto functions. It's possible to search for -related names and to list all names in a Python file and infer them. Jedi -understands docstrings and you can use Jedi autocompletion in your REPL as -well. - -Jedi uses a very simple API to connect with IDE's. There's a reference -implementation as a `VIM-Plugin `_, -which uses Jedi's autocompletion. We encourage you to use Jedi in your IDEs. -It's really easy. - -To give you a simple example how you can use the Jedi library, here is an -example for the autocompletion feature: - ->>> import jedi ->>> source = ''' -... import datetime -... datetime.da''' ->>> script = jedi.Script(source, 3, len('datetime.da'), 'example.py') ->>> script - ->>> completions = script.completions() ->>> completions #doctest: +ELLIPSIS -[, , ...] ->>> print(completions[0].complete) -te ->>> print(completions[0].name) -date - -As you see Jedi is pretty simple and allows you to concentrate on writing a -good text editor, while still having very good IDE features for Python. -""" - -__version__ = '0.10.0' - -from jedi.api import Script, Interpreter, NotFoundError, set_debug_function -from jedi.api import preload_module, defined_names, names -from jedi import settings diff --git a/pythonFiles/preview/jedi/_compatibility.py b/pythonFiles/preview/jedi/_compatibility.py deleted file mode 100644 index 352c2d6b5da9..000000000000 --- a/pythonFiles/preview/jedi/_compatibility.py +++ /dev/null @@ -1,267 +0,0 @@ -""" -To ensure compatibility from Python ``2.6`` - ``3.3``, a module has been -created. Clearly there is huge need to use conforming syntax. -""" -import sys -import imp -import os -import re -import pkgutil -try: - import importlib -except ImportError: - pass - -# Cannot use sys.version.major and minor names, because in Python 2.6 it's not -# a namedtuple. -is_py3 = sys.version_info[0] >= 3 -is_py33 = is_py3 and sys.version_info[1] >= 3 -is_py34 = is_py3 and sys.version_info[1] >= 4 -is_py35 = is_py3 and sys.version_info[1] >= 5 -is_py26 = not is_py3 and sys.version_info[1] < 7 -py_version = int(str(sys.version_info[0]) + str(sys.version_info[1])) - - -class DummyFile(object): - def __init__(self, loader, string): - self.loader = loader - self.string = string - - def read(self): - return self.loader.get_source(self.string) - - def close(self): - del self.loader - - -def find_module_py33(string, path=None): - loader = importlib.machinery.PathFinder.find_module(string, path) - - if loader is None and path is None: # Fallback to find builtins - try: - loader = importlib.find_loader(string) - except ValueError as e: - # See #491. Importlib might raise a ValueError, to avoid this, we - # just raise an ImportError to fix the issue. - raise ImportError("Originally " + repr(e)) - - if loader is None: - raise ImportError("Couldn't find a loader for {0}".format(string)) - - try: - is_package = loader.is_package(string) - if is_package: - if hasattr(loader, 'path'): - module_path = os.path.dirname(loader.path) - else: - # At least zipimporter does not have path attribute - module_path = os.path.dirname(loader.get_filename(string)) - if hasattr(loader, 'archive'): - module_file = DummyFile(loader, string) - else: - module_file = None - else: - module_path = loader.get_filename(string) - module_file = DummyFile(loader, string) - except AttributeError: - # ExtensionLoader has not attribute get_filename, instead it has a - # path attribute that we can use to retrieve the module path - try: - module_path = loader.path - module_file = DummyFile(loader, string) - except AttributeError: - module_path = string - module_file = None - finally: - is_package = False - - if hasattr(loader, 'archive'): - module_path = loader.archive - - return module_file, module_path, is_package - - -def find_module_pre_py33(string, path=None): - try: - module_file, module_path, description = imp.find_module(string, path) - module_type = description[2] - return module_file, module_path, module_type is imp.PKG_DIRECTORY - except ImportError: - pass - - if path is None: - path = sys.path - for item in path: - loader = pkgutil.get_importer(item) - if loader: - try: - loader = loader.find_module(string) - if loader: - is_package = loader.is_package(string) - is_archive = hasattr(loader, 'archive') - try: - module_path = loader.get_filename(string) - except AttributeError: - # fallback for py26 - try: - module_path = loader._get_filename(string) - except AttributeError: - continue - if is_package: - module_path = os.path.dirname(module_path) - if is_archive: - module_path = loader.archive - file = None - if not is_package or is_archive: - file = DummyFile(loader, string) - return (file, module_path, is_package) - except ImportError: - pass - raise ImportError("No module named {0}".format(string)) - - -find_module = find_module_py33 if is_py33 else find_module_pre_py33 -find_module.__doc__ = """ -Provides information about a module. - -This function isolates the differences in importing libraries introduced with -python 3.3 on; it gets a module name and optionally a path. It will return a -tuple containin an open file for the module (if not builtin), the filename -or the name of the module if it is a builtin one and a boolean indicating -if the module is contained in a package. -""" - - -# unicode function -try: - unicode = unicode -except NameError: - unicode = str - -if is_py3: - u = lambda s: s -else: - u = lambda s: s.decode('utf-8') - -u.__doc__ = """ -Decode a raw string into unicode object. Do nothing in Python 3. -""" - -# exec function -if is_py3: - def exec_function(source, global_map): - exec(source, global_map) -else: - eval(compile("""def exec_function(source, global_map): - exec source in global_map """, 'blub', 'exec')) - -# re-raise function -if is_py3: - def reraise(exception, traceback): - raise exception.with_traceback(traceback) -else: - eval(compile(""" -def reraise(exception, traceback): - raise exception, None, traceback -""", 'blub', 'exec')) - -reraise.__doc__ = """ -Re-raise `exception` with a `traceback` object. - -Usage:: - - reraise(Exception, sys.exc_info()[2]) - -""" - -class Python3Method(object): - def __init__(self, func): - self.func = func - - def __get__(self, obj, objtype): - if obj is None: - return lambda *args, **kwargs: self.func(*args, **kwargs) - else: - return lambda *args, **kwargs: self.func(obj, *args, **kwargs) - - -def use_metaclass(meta, *bases): - """ Create a class with a metaclass. """ - if not bases: - bases = (object,) - return meta("HackClass", bases, {}) - - -try: - encoding = sys.stdout.encoding - if encoding is None: - encoding = 'utf-8' -except AttributeError: - encoding = 'ascii' - - -def u(string): - """Cast to unicode DAMMIT! - Written because Python2 repr always implicitly casts to a string, so we - have to cast back to a unicode (and we now that we always deal with valid - unicode, because we check that in the beginning). - """ - if is_py3: - return str(string) - - if not isinstance(string, unicode): - return unicode(str(string), 'UTF-8') - return string - -try: - import builtins # module name in python 3 -except ImportError: - import __builtin__ as builtins - - -import ast - - -def literal_eval(string): - # py3.0, py3.1 and py32 don't support unicode literals. Support those, I - # don't want to write two versions of the tokenizer. - if is_py3 and sys.version_info.minor < 3: - if re.match('[uU][\'"]', string): - string = string[1:] - return ast.literal_eval(string) - - -try: - from itertools import zip_longest -except ImportError: - from itertools import izip_longest as zip_longest # Python 2 - - -def no_unicode_pprint(dct): - """ - Python 2/3 dict __repr__ may be different, because of unicode differens - (with or without a `u` prefix). Normally in doctests we could use `pprint` - to sort dicts and check for equality, but here we have to write a separate - function to do that. - """ - import pprint - s = pprint.pformat(dct) - print(re.sub("u'", "'", s)) - - -def utf8_repr(func): - """ - ``__repr__`` methods in Python 2 don't allow unicode objects to be - returned. Therefore cast them to utf-8 bytes in this decorator. - """ - def wrapper(self): - result = func(self) - if isinstance(result, unicode): - return result.encode('utf-8') - else: - return result - - if is_py3: - return func - else: - return wrapper diff --git a/pythonFiles/preview/jedi/api/__init__.py b/pythonFiles/preview/jedi/api/__init__.py deleted file mode 100644 index ab91934be6bd..000000000000 --- a/pythonFiles/preview/jedi/api/__init__.py +++ /dev/null @@ -1,467 +0,0 @@ -""" -The API basically only provides one class. You can create a :class:`Script` and -use its methods. - -Additionally you can add a debug function with :func:`set_debug_function`. -Alternatively, if you don't need a custom function and are happy with printing -debug messages to stdout, simply call :func:`set_debug_function` without -arguments. - -.. warning:: Please, note that Jedi is **not thread safe**. -""" -import os -import warnings -import sys - -from jedi._compatibility import unicode -from jedi.parser import load_grammar -from jedi.parser import tree -from jedi.parser.fast import FastParser -from jedi.parser.utils import save_parser -from jedi import debug -from jedi import settings -from jedi import common -from jedi import cache -from jedi.api import classes -from jedi.api import interpreter -from jedi.api import usages -from jedi.api import helpers -from jedi.api.completion import Completion -from jedi.evaluate import Evaluator -from jedi.evaluate import representation as er -from jedi.evaluate import imports -from jedi.evaluate.param import try_iter_content -from jedi.evaluate.helpers import get_module_names -from jedi.evaluate.sys_path import get_venv_path -from jedi.evaluate.iterable import unpack_tuple_to_dict - -# Jedi uses lots and lots of recursion. By setting this a little bit higher, we -# can remove some "maximum recursion depth" errors. -sys.setrecursionlimit(2000) - - -class NotFoundError(Exception): - """A custom error to avoid catching the wrong exceptions. - - .. deprecated:: 0.9.0 - Not in use anymore, Jedi just returns no goto result if you're not on a - valid name. - .. todo:: Remove! - """ - - -class Script(object): - """ - A Script is the base for completions, goto or whatever you want to do with - |jedi|. - - You can either use the ``source`` parameter or ``path`` to read a file. - Usually you're going to want to use both of them (in an editor). - - The script might be analyzed in a different ``sys.path`` than |jedi|: - - - if `sys_path` parameter is not ``None``, it will be used as ``sys.path`` - for the script; - - - if `sys_path` parameter is ``None`` and ``VIRTUAL_ENV`` environment - variable is defined, ``sys.path`` for the specified environment will be - guessed (see :func:`jedi.evaluate.sys_path.get_venv_path`) and used for - the script; - - - otherwise ``sys.path`` will match that of |jedi|. - - :param source: The source code of the current file, separated by newlines. - :type source: str - :param line: The line to perform actions on (starting with 1). - :type line: int - :param column: The column of the cursor (starting with 0). - :type column: int - :param path: The path of the file in the file system, or ``''`` if - it hasn't been saved yet. - :type path: str or None - :param encoding: The encoding of ``source``, if it is not a - ``unicode`` object (default ``'utf-8'``). - :type encoding: str - :param source_encoding: The encoding of ``source``, if it is not a - ``unicode`` object (default ``'utf-8'``). - :type encoding: str - :param sys_path: ``sys.path`` to use during analysis of the script - :type sys_path: list - - """ - def __init__(self, source=None, line=None, column=None, path=None, - encoding='utf-8', source_path=None, source_encoding=None, - sys_path=None): - if source_path is not None: - warnings.warn("Use path instead of source_path.", DeprecationWarning) - path = source_path - if source_encoding is not None: - warnings.warn("Use encoding instead of source_encoding.", DeprecationWarning) - encoding = source_encoding - - self._orig_path = path - # An empty path (also empty string) should always result in no path. - self.path = os.path.abspath(path) if path else None - - if source is None: - # TODO add a better warning than the traceback! - try: - with open(path) as f: - source = f.read() - except UnicodeDecodeError: - with open(path, encoding=encoding) as f: - source = f.read() - - self._source = common.source_to_unicode(source, encoding) - self._code_lines = common.splitlines(self._source) - line = max(len(self._code_lines), 1) if line is None else line - if not (0 < line <= len(self._code_lines)): - raise ValueError('`line` parameter is not in a valid range.') - - line_len = len(self._code_lines[line - 1]) - column = line_len if column is None else column - if not (0 <= column <= line_len): - raise ValueError('`column` parameter is not in a valid range.') - self._pos = line, column - self._path = path - - cache.clear_time_caches() - debug.reset_time() - self._grammar = load_grammar(version='%s.%s' % sys.version_info[:2]) - if sys_path is None: - venv = os.getenv('VIRTUAL_ENV') - if venv: - sys_path = list(get_venv_path(venv)) - self._evaluator = Evaluator(self._grammar, sys_path=sys_path) - debug.speed('init') - - def _get_module(self): - cache.invalidate_star_import_cache(self._path) - parser = FastParser(self._grammar, self._source, self.path) - save_parser(self.path, parser, pickling=False) - - module = self._evaluator.wrap(parser.module) - imports.add_module(self._evaluator, unicode(module.name), module) - return parser.module - - @property - def source_path(self): - """ - .. deprecated:: 0.7.0 - Use :attr:`.path` instead. - .. todo:: Remove! - """ - warnings.warn("Use path instead of source_path.", DeprecationWarning) - return self.path - - def __repr__(self): - return '<%s: %s>' % (self.__class__.__name__, repr(self._orig_path)) - - def completions(self): - """ - Return :class:`classes.Completion` objects. Those objects contain - information about the completions, more than just names. - - :return: Completion objects, sorted by name and __ comes last. - :rtype: list of :class:`classes.Completion` - """ - debug.speed('completions start') - completion = Completion( - self._evaluator, self._get_module(), self._code_lines, - self._pos, self.call_signatures - ) - completions = completion.completions() - debug.speed('completions end') - return completions - - def goto_definitions(self): - """ - Return the definitions of a the path under the cursor. goto function! - This follows complicated paths and returns the end, not the first - definition. The big difference between :meth:`goto_assignments` and - :meth:`goto_definitions` is that :meth:`goto_assignments` doesn't - follow imports and statements. Multiple objects may be returned, - because Python itself is a dynamic language, which means depending on - an option you can have two different versions of a function. - - :rtype: list of :class:`classes.Definition` - """ - leaf = self._get_module().name_for_position(self._pos) - if leaf is None: - leaf = self._get_module().get_leaf_for_position(self._pos) - if leaf is None: - return [] - definitions = helpers.evaluate_goto_definition(self._evaluator, leaf) - - names = [s.name for s in definitions] - defs = [classes.Definition(self._evaluator, name) for name in names] - # The additional set here allows the definitions to become unique in an - # API sense. In the internals we want to separate more things than in - # the API. - return helpers.sorted_definitions(set(defs)) - - def goto_assignments(self, follow_imports=False): - """ - Return the first definition found, while optionally following imports. - Multiple objects may be returned, because Python itself is a - dynamic language, which means depending on an option you can have two - different versions of a function. - - :rtype: list of :class:`classes.Definition` - """ - def filter_follow_imports(names): - for name in names: - definition = name.get_definition() - if definition.type in ('import_name', 'import_from'): - imp = imports.ImportWrapper(self._evaluator, name) - for name in filter_follow_imports(imp.follow(is_goto=True)): - yield name - else: - yield name - - names = self._goto() - if follow_imports: - names = filter_follow_imports(names) - - defs = [classes.Definition(self._evaluator, d) for d in set(names)] - return helpers.sorted_definitions(defs) - - def _goto(self): - """ - Used for goto_assignments and usages. - """ - name = self._get_module().name_for_position(self._pos) - if name is None: - return [] - return list(self._evaluator.goto(name)) - - def usages(self, additional_module_paths=()): - """ - Return :class:`classes.Definition` objects, which contain all - names that point to the definition of the name under the cursor. This - is very useful for refactoring (renaming), or to show all usages of a - variable. - - .. todo:: Implement additional_module_paths - - :rtype: list of :class:`classes.Definition` - """ - temp, settings.dynamic_flow_information = \ - settings.dynamic_flow_information, False - try: - user_stmt = self._get_module().get_statement_for_position(self._pos) - definitions = self._goto() - if not definitions and isinstance(user_stmt, tree.Import): - # For not defined imports (goto doesn't find something, we take - # the name as a definition. This is enough, because every name - # points to it. - name = user_stmt.name_for_position(self._pos) - if name is None: - # Must be syntax - return [] - definitions = [name] - - if not definitions: - # Without a definition for a name we cannot find references. - return [] - - if not isinstance(user_stmt, tree.Import): - # import case is looked at with add_import_name option - definitions = usages.usages_add_import_modules(self._evaluator, - definitions) - - module = set([d.get_parent_until() for d in definitions]) - module.add(self._get_module()) - names = usages.usages(self._evaluator, definitions, module) - - for d in set(definitions): - names.append(classes.Definition(self._evaluator, d)) - finally: - settings.dynamic_flow_information = temp - - return helpers.sorted_definitions(set(names)) - - def call_signatures(self): - """ - Return the function object of the call you're currently in. - - E.g. if the cursor is here:: - - abs(# <-- cursor is here - - This would return the ``abs`` function. On the other hand:: - - abs()# <-- cursor is here - - This would return an empty list.. - - :rtype: list of :class:`classes.CallSignature` - """ - call_signature_details = \ - helpers.get_call_signature_details(self._get_module(), self._pos) - if call_signature_details is None: - return [] - - with common.scale_speed_settings(settings.scale_call_signatures): - definitions = helpers.cache_call_signatures( - self._evaluator, - call_signature_details.bracket_leaf, - self._code_lines, - self._pos - ) - debug.speed('func_call followed') - - return [classes.CallSignature(self._evaluator, d.name, - call_signature_details.bracket_leaf.start_pos, - call_signature_details.call_index, - call_signature_details.keyword_name_str) - for d in definitions if hasattr(d, 'py__call__')] - - def _analysis(self): - self._evaluator.is_analysis = True - self._evaluator.analysis_modules = [self._get_module()] - try: - for node in self._get_module().nodes_to_execute(): - if node.type in ('funcdef', 'classdef'): - if node.type == 'classdef': - continue - raise NotImplementedError - er.Function(self._evaluator, node).get_decorated_func() - elif isinstance(node, tree.Import): - import_names = set(node.get_defined_names()) - if node.is_nested(): - import_names |= set(path[-1] for path in node.paths()) - for n in import_names: - imports.ImportWrapper(self._evaluator, n).follow() - elif node.type == 'expr_stmt': - types = self._evaluator.eval_element(node) - for testlist in node.children[:-1:2]: - # Iterate tuples. - unpack_tuple_to_dict(self._evaluator, types, testlist) - else: - try_iter_content(self._evaluator.goto_definitions(node)) - self._evaluator.reset_recursion_limitations() - - ana = [a for a in self._evaluator.analysis if self.path == a.path] - return sorted(set(ana), key=lambda x: x.line) - finally: - self._evaluator.is_analysis = False - - -class Interpreter(Script): - """ - Jedi API for Python REPLs. - - In addition to completion of simple attribute access, Jedi - supports code completion based on static code analysis. - Jedi can complete attributes of object which is not initialized - yet. - - >>> from os.path import join - >>> namespace = locals() - >>> script = Interpreter('join("").up', [namespace]) - >>> print(script.completions()[0].name) - upper - """ - - def __init__(self, source, namespaces, **kwds): - """ - Parse `source` and mixin interpreted Python objects from `namespaces`. - - :type source: str - :arg source: Code to parse. - :type namespaces: list of dict - :arg namespaces: a list of namespace dictionaries such as the one - returned by :func:`locals`. - - Other optional arguments are same as the ones for :class:`Script`. - If `line` and `column` are None, they are assumed be at the end of - `source`. - """ - try: - namespaces = [dict(n) for n in namespaces] - except Exception: - raise TypeError("namespaces must be a non-empty list of dicts.") - - super(Interpreter, self).__init__(source, **kwds) - self.namespaces = namespaces - - parser_module = super(Interpreter, self)._get_module() - self._module = interpreter.MixedModule(self._evaluator, parser_module, self.namespaces) - - def _get_module(self): - return self._module - - -def defined_names(source, path=None, encoding='utf-8'): - """ - Get all definitions in `source` sorted by its position. - - This functions can be used for listing functions, classes and - data defined in a file. This can be useful if you want to list - them in "sidebar". Each element in the returned list also has - `defined_names` method which can be used to get sub-definitions - (e.g., methods in class). - - :rtype: list of classes.Definition - - .. deprecated:: 0.9.0 - Use :func:`names` instead. - .. todo:: Remove! - """ - warnings.warn("Use call_signatures instead.", DeprecationWarning) - return names(source, path, encoding) - - -def names(source=None, path=None, encoding='utf-8', all_scopes=False, - definitions=True, references=False): - """ - Returns a list of `Definition` objects, containing name parts. - This means you can call ``Definition.goto_assignments()`` and get the - reference of a name. - The parameters are the same as in :py:class:`Script`, except or the - following ones: - - :param all_scopes: If True lists the names of all scopes instead of only - the module namespace. - :param definitions: If True lists the names that have been defined by a - class, function or a statement (``a = b`` returns ``a``). - :param references: If True lists all the names that are not listed by - ``definitions=True``. E.g. ``a = b`` returns ``b``. - """ - def def_ref_filter(_def): - is_def = _def.is_definition() - return definitions and is_def or references and not is_def - - # Set line/column to a random position, because they don't matter. - script = Script(source, line=1, column=0, path=path, encoding=encoding) - defs = [classes.Definition(script._evaluator, name_part) - for name_part in get_module_names(script._get_module(), all_scopes)] - return sorted(filter(def_ref_filter, defs), key=lambda x: (x.line, x.column)) - - -def preload_module(*modules): - """ - Preloading modules tells Jedi to load a module now, instead of lazy parsing - of modules. Usful for IDEs, to control which modules to load on startup. - - :param modules: different module names, list of string. - """ - for m in modules: - s = "import %s as x; x." % m - Script(s, 1, len(s), None).completions() - - -def set_debug_function(func_cb=debug.print_to_stdout, warnings=True, - notices=True, speed=True): - """ - Define a callback debug function to get all the debug messages. - - If you don't specify any arguments, debug messages will be printed to stdout. - - :param func_cb: The callback function for debug messages, with n params. - """ - debug.debug_function = func_cb - debug.enable_warning = warnings - debug.enable_notice = notices - debug.enable_speed = speed diff --git a/pythonFiles/preview/jedi/api/classes.py b/pythonFiles/preview/jedi/api/classes.py deleted file mode 100644 index a1d3ce8a3412..000000000000 --- a/pythonFiles/preview/jedi/api/classes.py +++ /dev/null @@ -1,764 +0,0 @@ -""" -The :mod:`jedi.api.classes` module contains the return classes of the API. -These classes are the much bigger part of the whole API, because they contain -the interesting information about completion and goto operations. -""" -import warnings -from itertools import chain -import re - -from jedi._compatibility import unicode, use_metaclass -from jedi import settings -from jedi import common -from jedi.parser import tree -from jedi.parser.utils import load_parser -from jedi.evaluate.cache import memoize_default, CachedMetaClass -from jedi.evaluate import representation as er -from jedi.evaluate import iterable -from jedi.evaluate import imports -from jedi.evaluate import compiled -from jedi.evaluate.compiled import mixed -from jedi.api import keywords -from jedi.evaluate.finder import filter_definition_names - - -def defined_names(evaluator, scope): - """ - List sub-definitions (e.g., methods in class). - - :type scope: Scope - :rtype: list of Definition - """ - dct = scope.names_dict - names = list(chain.from_iterable(dct.values())) - names = filter_definition_names(names, scope) - return [Definition(evaluator, d) for d in sorted(names, key=lambda s: s.start_pos)] - - -class BaseDefinition(object): - _mapping = { - 'posixpath': 'os.path', - 'riscospath': 'os.path', - 'ntpath': 'os.path', - 'os2emxpath': 'os.path', - 'macpath': 'os.path', - 'genericpath': 'os.path', - 'posix': 'os', - '_io': 'io', - '_functools': 'functools', - '_sqlite3': 'sqlite3', - '__builtin__': '', - 'builtins': '', - } - - _tuple_mapping = dict((tuple(k.split('.')), v) for (k, v) in { - 'argparse._ActionsContainer': 'argparse.ArgumentParser', - '_sre.SRE_Match': 're.MatchObject', - '_sre.SRE_Pattern': 're.RegexObject', - }.items()) - - def __init__(self, evaluator, name): - self._evaluator = evaluator - self._name = name - """ - An instance of :class:`jedi.parser.reprsentation.Name` subclass. - """ - self._definition = evaluator.wrap(self._name.get_definition()) - self.is_keyword = isinstance(self._definition, keywords.Keyword) - - # generate a path to the definition - self._module = name.get_parent_until() - if self.in_builtin_module(): - self.module_path = None - else: - self.module_path = self._module.path - """Shows the file path of a module. e.g. ``/usr/lib/python2.7/os.py``""" - - @property - def name(self): - """ - Name of variable/function/class/module. - - For example, for ``x = None`` it returns ``'x'``. - - :rtype: str or None - """ - return unicode(self._name) - - @property - def start_pos(self): - """ - .. deprecated:: 0.7.0 - Use :attr:`.line` and :attr:`.column` instead. - .. todo:: Remove! - """ - warnings.warn("Use line/column instead.", DeprecationWarning) - return self._name.start_pos - - @property - def type(self): - """ - The type of the definition. - - Here is an example of the value of this attribute. Let's consider - the following source. As what is in ``variable`` is unambiguous - to Jedi, :meth:`jedi.Script.goto_definitions` should return a list of - definition for ``sys``, ``f``, ``C`` and ``x``. - - >>> from jedi import Script - >>> source = ''' - ... import keyword - ... - ... class C: - ... pass - ... - ... class D: - ... pass - ... - ... x = D() - ... - ... def f(): - ... pass - ... - ... for variable in [keyword, f, C, x]: - ... variable''' - - >>> script = Script(source) - >>> defs = script.goto_definitions() - - Before showing what is in ``defs``, let's sort it by :attr:`line` - so that it is easy to relate the result to the source code. - - >>> defs = sorted(defs, key=lambda d: d.line) - >>> defs # doctest: +NORMALIZE_WHITESPACE - [, , - , ] - - Finally, here is what you can get from :attr:`type`: - - >>> defs[0].type - 'module' - >>> defs[1].type - 'class' - >>> defs[2].type - 'instance' - >>> defs[3].type - 'function' - - """ - stripped = self._definition - if isinstance(stripped, er.InstanceElement): - stripped = stripped.var - - if isinstance(stripped, (compiled.CompiledObject, mixed.MixedObject)): - return stripped.api_type() - elif isinstance(stripped, iterable.Array): - return 'instance' - elif isinstance(stripped, tree.Import): - return 'import' - - string = type(stripped).__name__.lower().replace('wrapper', '') - if string == 'exprstmt': - return 'statement' - else: - return string - - def _path(self): - """The path to a module/class/function definition.""" - path = [] - par = self._definition - while par is not None: - if isinstance(par, tree.Import): - path += imports.ImportWrapper(self._evaluator, self._name).import_path - break - try: - name = par.name - except AttributeError: - pass - else: - if isinstance(par, er.ModuleWrapper): - # TODO just make the path dotted from the beginning, we - # shouldn't really split here. - path[0:0] = par.py__name__().split('.') - break - else: - path.insert(0, unicode(name)) - par = par.parent - return path - - @property - def module_name(self): - """ - The module name. - - >>> from jedi import Script - >>> source = 'import json' - >>> script = Script(source, path='example.py') - >>> d = script.goto_definitions()[0] - >>> print(d.module_name) # doctest: +ELLIPSIS - json - """ - return str(self._module.name) - - def in_builtin_module(self): - """Whether this is a builtin module.""" - return isinstance(self._module, compiled.CompiledObject) - - @property - def line(self): - """The line where the definition occurs (starting with 1).""" - if self.in_builtin_module(): - return None - return self._name.start_pos[0] - - @property - def column(self): - """The column where the definition occurs (starting with 0).""" - if self.in_builtin_module(): - return None - return self._name.start_pos[1] - - def docstring(self, raw=False): - r""" - Return a document string for this completion object. - - Example: - - >>> from jedi import Script - >>> source = '''\ - ... def f(a, b=1): - ... "Document for function f." - ... ''' - >>> script = Script(source, 1, len('def f'), 'example.py') - >>> doc = script.goto_definitions()[0].docstring() - >>> print(doc) - f(a, b=1) - - Document for function f. - - Notice that useful extra information is added to the actual - docstring. For function, it is call signature. If you need - actual docstring, use ``raw=True`` instead. - - >>> print(script.goto_definitions()[0].docstring(raw=True)) - Document for function f. - - """ - if raw: - return _Help(self._definition).raw() - else: - return _Help(self._definition).full() - - @property - def doc(self): - """ - .. deprecated:: 0.8.0 - Use :meth:`.docstring` instead. - .. todo:: Remove! - """ - warnings.warn("Use docstring() instead.", DeprecationWarning) - return self.docstring() - - @property - def raw_doc(self): - """ - .. deprecated:: 0.8.0 - Use :meth:`.docstring` instead. - .. todo:: Remove! - """ - warnings.warn("Use docstring() instead.", DeprecationWarning) - return self.docstring(raw=True) - - @property - def description(self): - """A textual description of the object.""" - return unicode(self._name) - - @property - def full_name(self): - """ - Dot-separated path of this object. - - It is in the form of ``[.[...]][.]``. - It is useful when you want to look up Python manual of the - object at hand. - - Example: - - >>> from jedi import Script - >>> source = ''' - ... import os - ... os.path.join''' - >>> script = Script(source, 3, len('os.path.join'), 'example.py') - >>> print(script.goto_definitions()[0].full_name) - os.path.join - - Notice that it correctly returns ``'os.path.join'`` instead of - (for example) ``'posixpath.join'``. - - """ - path = [unicode(p) for p in self._path()] - # TODO add further checks, the mapping should only occur on stdlib. - if not path: - return None # for keywords the path is empty - - with common.ignored(KeyError): - path[0] = self._mapping[path[0]] - for key, repl in self._tuple_mapping.items(): - if tuple(path[:len(key)]) == key: - path = [repl] + path[len(key):] - - return '.'.join(path if path[0] else path[1:]) - - def goto_assignments(self): - defs = self._evaluator.goto(self._name) - return [Definition(self._evaluator, d) for d in defs] - - @memoize_default() - def _follow_statements_imports(self): - """ - Follow both statements and imports, as far as possible. - """ - if self._definition.isinstance(tree.ExprStmt): - return self._evaluator.eval_statement(self._definition) - elif self._definition.isinstance(tree.Import): - return imports.ImportWrapper(self._evaluator, self._name).follow() - else: - return set([self._definition]) - - @property - @memoize_default() - def params(self): - """ - Raises an ``AttributeError``if the definition is not callable. - Otherwise returns a list of `Definition` that represents the params. - """ - followed = list(self._follow_statements_imports()) - if not followed or not hasattr(followed[0], 'py__call__'): - raise AttributeError() - followed = followed[0] # only check the first one. - - if followed.type in ('funcdef', 'lambda'): - if isinstance(followed, er.InstanceElement): - params = followed.params[1:] - else: - params = followed.params - elif followed.isinstance(er.compiled.CompiledObject): - params = followed.params - elif isinstance(followed, er.Class): - try: - sub = followed.get_subscope_by_name('__init__') - params = sub.params[1:] # ignore self - except KeyError: - return [] - elif isinstance(followed, er.Instance): - try: - sub = followed.get_subscope_by_name('__call__') - params = sub.params[1:] # ignore self - except KeyError: - return [] - else: - return [] - return [_Param(self._evaluator, p.name) for p in params] - - def parent(self): - scope = self._definition.get_parent_scope() - scope = self._evaluator.wrap(scope) - return Definition(self._evaluator, scope.name) - - def __repr__(self): - return "<%s %s>" % (type(self).__name__, self.description) - - def get_line_code(self, before=0, after=0): - """ - Returns the line of code where this object was defined. - - :param before: Add n lines before the current line to the output. - :param after: Add n lines after the current line to the output. - - :return str: Returns the line(s) of code or an empty string if it's a - builtin. - """ - if self.in_builtin_module(): - return '' - - path = self._definition.get_parent_until().path - parser = load_parser(path) - lines = common.splitlines(parser.source) - - line_nr = self._name.start_pos[0] - start_line_nr = line_nr - before - return '\n'.join(lines[start_line_nr:line_nr + after + 1]) - - -class Completion(BaseDefinition): - """ - `Completion` objects are returned from :meth:`api.Script.completions`. They - provide additional information about a completion. - """ - def __init__(self, evaluator, name, stack, like_name_length): - super(Completion, self).__init__(evaluator, name) - - self._like_name_length = like_name_length - self._stack = stack - - # Completion objects with the same Completion name (which means - # duplicate items in the completion) - self._same_name_completions = [] - - def _complete(self, like_name): - append = '' - if settings.add_bracket_after_function \ - and self.type == 'Function': - append = '(' - - if isinstance(self._definition, tree.Param) and self._stack is not None: - node_names = list(self._stack.get_node_names(self._evaluator.grammar)) - if 'trailer' in node_names and 'argument' not in node_names: - append += '=' - - name = str(self._name) - if like_name: - name = name[self._like_name_length:] - return name + append - - @property - def complete(self): - """ - Return the rest of the word, e.g. completing ``isinstance``:: - - isinstan# <-- Cursor is here - - would return the string 'ce'. It also adds additional stuff, depending - on your `settings.py`. - """ - return self._complete(True) - - @property - def name_with_symbols(self): - """ - Similar to :attr:`name`, but like :attr:`name` - returns also the symbols, for example:: - - list() - - would return ``.append`` and others (which means it adds a dot). - """ - return self._complete(False) - - @property - def description(self): - """Provide a description of the completion object.""" - if self._definition is None: - return '' - t = self.type - if t == 'statement' or t == 'import': - desc = self._definition.get_code() - else: - desc = '.'.join(unicode(p) for p in self._path()) - - line = '' if self.in_builtin_module else '@%s' % self.line - return '%s: %s%s' % (t, desc, line) - - def __repr__(self): - return '<%s: %s>' % (type(self).__name__, self._name) - - def docstring(self, raw=False, fast=True): - """ - :param fast: Don't follow imports that are only one level deep like - ``import foo``, but follow ``from foo import bar``. This makes - sense for speed reasons. Completing `import a` is slow if you use - the ``foo.docstring(fast=False)`` on every object, because it - parses all libraries starting with ``a``. - """ - definition = self._definition - if isinstance(definition, tree.Import): - i = imports.ImportWrapper(self._evaluator, self._name) - if len(i.import_path) > 1 or not fast: - followed = self._follow_statements_imports() - if followed: - # TODO: Use all of the followed objects as input to Documentation. - definition = list(followed)[0] - - if raw: - return _Help(definition).raw() - else: - return _Help(definition).full() - - @property - def type(self): - """ - The type of the completion objects. Follows imports. For a further - description, look at :attr:`jedi.api.classes.BaseDefinition.type`. - """ - if isinstance(self._definition, tree.Import): - i = imports.ImportWrapper(self._evaluator, self._name) - if len(i.import_path) <= 1: - return 'module' - - followed = self.follow_definition() - if followed: - # Caveat: Only follows the first one, ignore the other ones. - # This is ok, since people are almost never interested in - # variations. - return followed[0].type - return super(Completion, self).type - - @memoize_default() - def _follow_statements_imports(self): - # imports completion is very complicated and needs to be treated - # separately in Completion. - definition = self._definition - if definition.isinstance(tree.Import): - i = imports.ImportWrapper(self._evaluator, self._name) - return i.follow() - return super(Completion, self)._follow_statements_imports() - - @memoize_default() - def follow_definition(self): - """ - Return the original definitions. I strongly recommend not using it for - your completions, because it might slow down |jedi|. If you want to - read only a few objects (<=20), it might be useful, especially to get - the original docstrings. The basic problem of this function is that it - follows all results. This means with 1000 completions (e.g. numpy), - it's just PITA-slow. - """ - defs = self._follow_statements_imports() - return [Definition(self._evaluator, d.name) for d in defs] - - -class Definition(use_metaclass(CachedMetaClass, BaseDefinition)): - """ - *Definition* objects are returned from :meth:`api.Script.goto_assignments` - or :meth:`api.Script.goto_definitions`. - """ - def __init__(self, evaluator, definition): - super(Definition, self).__init__(evaluator, definition) - - @property - def description(self): - """ - A description of the :class:`.Definition` object, which is heavily used - in testing. e.g. for ``isinstance`` it returns ``def isinstance``. - - Example: - - >>> from jedi import Script - >>> source = ''' - ... def f(): - ... pass - ... - ... class C: - ... pass - ... - ... variable = f if random.choice([0,1]) else C''' - >>> script = Script(source, column=3) # line is maximum by default - >>> defs = script.goto_definitions() - >>> defs = sorted(defs, key=lambda d: d.line) - >>> defs - [, ] - >>> str(defs[0].description) # strip literals in python2 - 'def f' - >>> str(defs[1].description) - 'class C' - - """ - d = self._definition - if isinstance(d, er.InstanceElement): - d = d.var - - if isinstance(d, compiled.CompiledObject): - typ = d.api_type() - if typ == 'instance': - typ = 'class' # The description should be similar to Py objects. - d = typ + ' ' + d.name.get_code() - elif isinstance(d, iterable.Array): - d = 'class ' + d.type - elif isinstance(d, (tree.Class, er.Class, er.Instance)): - d = 'class ' + unicode(d.name) - elif isinstance(d, (er.Function, tree.Function)): - d = 'def ' + unicode(d.name) - elif isinstance(d, tree.Module): - # only show module name - d = 'module %s' % self.module_name - elif isinstance(d, tree.Param): - d = d.get_code().strip() - if d.endswith(','): - d = d[:-1] # Remove the comma. - else: # ExprStmt - try: - first_leaf = d.first_leaf() - except AttributeError: - # `d` is already a Leaf (Name). - first_leaf = d - # Remove the prefix, because that's not what we want for get_code - # here. - old, first_leaf.prefix = first_leaf.prefix, '' - try: - d = d.get_code() - finally: - first_leaf.prefix = old - # Delete comments: - d = re.sub('#[^\n]+\n', ' ', d) - # Delete multi spaces/newlines - return re.sub('\s+', ' ', d).strip() - - @property - def desc_with_module(self): - """ - In addition to the definition, also return the module. - - .. warning:: Don't use this function yet, its behaviour may change. If - you really need it, talk to me. - - .. todo:: Add full path. This function is should return a - `module.class.function` path. - """ - position = '' if self.in_builtin_module else '@%s' % (self.line) - return "%s:%s%s" % (self.module_name, self.description, position) - - @memoize_default() - def defined_names(self): - """ - List sub-definitions (e.g., methods in class). - - :rtype: list of Definition - """ - defs = self._follow_statements_imports() - # For now we don't want base classes or evaluate decorators. - defs = [d.base if isinstance(d, (er.Class, er.Function)) else d for d in defs] - iterable = (defined_names(self._evaluator, d) for d in defs) - iterable = list(iterable) - return list(chain.from_iterable(iterable)) - - def is_definition(self): - """ - Returns True, if defined as a name in a statement, function or class. - Returns False, if it's a reference to such a definition. - """ - return self._name.is_definition() - - def __eq__(self, other): - return self._name.start_pos == other._name.start_pos \ - and self.module_path == other.module_path \ - and self.name == other.name \ - and self._evaluator == other._evaluator - - def __ne__(self, other): - return not self.__eq__(other) - - def __hash__(self): - return hash((self._name.start_pos, self.module_path, self.name, self._evaluator)) - - -class CallSignature(Definition): - """ - `CallSignature` objects is the return value of `Script.function_definition`. - It knows what functions you are currently in. e.g. `isinstance(` would - return the `isinstance` function. without `(` it would return nothing. - """ - def __init__(self, evaluator, executable_name, bracket_start_pos, index, key_name_str): - super(CallSignature, self).__init__(evaluator, executable_name) - self._index = index - self._key_name_str = key_name_str - self._bracket_start_pos = bracket_start_pos - - @property - def index(self): - """ - The Param index of the current call. - Returns None if the index cannot be found in the curent call. - """ - if self._key_name_str is not None: - for i, param in enumerate(self.params): - if self._key_name_str == param.name: - return i - if self.params and self.params[-1]._name.get_definition().stars == 2: - return i - else: - return None - - if self._index >= len(self.params): - - for i, param in enumerate(self.params): - # *args case - if param._name.get_definition().stars == 1: - return i - return None - return self._index - - @property - def bracket_start(self): - """ - The indent of the bracket that is responsible for the last function - call. - """ - return self._bracket_start_pos - - @property - def call_name(self): - """ - .. deprecated:: 0.8.0 - Use :attr:`.name` instead. - .. todo:: Remove! - - The name (e.g. 'isinstance') as a string. - """ - warnings.warn("Use name instead.", DeprecationWarning) - return unicode(self.name) - - @property - def module(self): - """ - .. deprecated:: 0.8.0 - Use :attr:`.module_name` for the module name. - .. todo:: Remove! - """ - return self._executable.get_parent_until() - - def __repr__(self): - return '<%s: %s index %s>' % (type(self).__name__, self._name, - self.index) - - -class _Param(Definition): - """ - Just here for backwards compatibility. - """ - def get_code(self): - """ - .. deprecated:: 0.8.0 - Use :attr:`.description` and :attr:`.name` instead. - .. todo:: Remove! - - A function to get the whole code of the param. - """ - warnings.warn("Use description instead.", DeprecationWarning) - return self.description - - -class _Help(object): - """ - Temporary implementation, will be used as `Script.help() or something in - the future. - """ - def __init__(self, definition): - self._name = definition - - def full(self): - try: - return self._name.doc - except AttributeError: - return self.raw() - - def raw(self): - """ - The raw docstring ``__doc__`` for any object. - - See :attr:`doc` for example. - """ - try: - return self._name.raw_doc - except AttributeError: - return '' diff --git a/pythonFiles/preview/jedi/api/completion.py b/pythonFiles/preview/jedi/api/completion.py deleted file mode 100644 index cc362dacbfa8..000000000000 --- a/pythonFiles/preview/jedi/api/completion.py +++ /dev/null @@ -1,256 +0,0 @@ -from itertools import chain - -from jedi.parser import token -from jedi.parser import tree -from jedi import debug -from jedi import settings -from jedi.api import classes -from jedi.api import helpers -from jedi.evaluate import imports -from jedi.api import keywords -from jedi.evaluate import compiled -from jedi.evaluate.helpers import call_of_leaf -from jedi.evaluate.finder import global_names_dict_generator, filter_definition_names - - -def get_call_signature_param_names(call_signatures): - # add named params - for call_sig in call_signatures: - # Allow protected access, because it's a public API. - module = call_sig._name.get_parent_until() - # Compiled modules typically don't allow keyword arguments. - if not isinstance(module, compiled.CompiledObject): - for p in call_sig.params: - # Allow access on _definition here, because it's a - # public API and we don't want to make the internal - # Name object public. - if p._definition.stars == 0: # no *args/**kwargs - yield p._name - - -def filter_names(evaluator, completion_names, stack, like_name): - comp_dct = {} - for name in set(completion_names): - if settings.case_insensitive_completion \ - and str(name).lower().startswith(like_name.lower()) \ - or str(name).startswith(like_name): - - if isinstance(name.parent, (tree.Function, tree.Class)): - # TODO I think this is a hack. It should be an - # er.Function/er.Class before that. - name = evaluator.wrap(name.parent).name - new = classes.Completion( - evaluator, - name, - stack, - len(like_name) - ) - k = (new.name, new.complete) # key - if k in comp_dct and settings.no_completion_duplicates: - comp_dct[k]._same_name_completions.append(new) - else: - comp_dct[k] = new - yield new - - -def get_user_scope(module, position): - """ - Returns the scope in which the user resides. This includes flows. - """ - user_stmt = module.get_statement_for_position(position) - if user_stmt is None: - def scan(scope): - for s in scope.children: - if s.start_pos <= position <= s.end_pos: - if isinstance(s, (tree.Scope, tree.Flow)): - return scan(s) or s - elif s.type in ('suite', 'decorated'): - return scan(s) - return None - - return scan(module) or module - else: - return user_stmt.get_parent_scope(include_flows=True) - - -class Completion: - def __init__(self, evaluator, module, code_lines, position, call_signatures_method): - self._evaluator = evaluator - self._module = evaluator.wrap(module) - self._code_lines = code_lines - - # The first step of completions is to get the name - self._like_name = helpers.get_on_completion_name(module, code_lines, position) - # The actual cursor position is not what we need to calculate - # everything. We want the start of the name we're on. - self._position = position[0], position[1] - len(self._like_name) - self._call_signatures_method = call_signatures_method - - def completions(self): - completion_names = self._get_context_completions() - - completions = filter_names(self._evaluator, completion_names, - self.stack, self._like_name) - - return sorted(completions, key=lambda x: (x.name.startswith('__'), - x.name.startswith('_'), - x.name.lower())) - - def _get_context_completions(self): - """ - Analyzes the context that a completion is made in and decides what to - return. - - Technically this works by generating a parser stack and analysing the - current stack for possible grammar nodes. - - Possible enhancements: - - global/nonlocal search global - - yield from / raise from <- could be only exceptions/generators - - In args: */**: no completion - - In params (also lambda): no completion before = - """ - - grammar = self._evaluator.grammar - - try: - self.stack = helpers.get_stack_at_position( - grammar, self._code_lines, self._module, self._position - ) - except helpers.OnErrorLeaf as e: - self.stack = None - if e.error_leaf.value == '.': - # After ErrorLeaf's that are dots, we will not do any - # completions since this probably just confuses the user. - return [] - # If we don't have a context, just use global completion. - - return self._global_completions() - - allowed_keywords, allowed_tokens = \ - helpers.get_possible_completion_types(grammar, self.stack) - - completion_names = list(self._get_keyword_completion_names(allowed_keywords)) - - if token.NAME in allowed_tokens: - # This means that we actually have to do type inference. - - symbol_names = list(self.stack.get_node_names(grammar)) - - nodes = list(self.stack.get_nodes()) - - if "import_stmt" in symbol_names: - level = 0 - only_modules = True - level, names = self._parse_dotted_names(nodes) - if "import_from" in symbol_names: - if 'import' in nodes: - only_modules = False - else: - assert "import_name" in symbol_names - - completion_names += self._get_importer_names( - names, - level, - only_modules - ) - elif nodes and nodes[-1] in ('as', 'def', 'class'): - # No completions for ``with x as foo`` and ``import x as foo``. - # Also true for defining names as a class or function. - return list(self._get_class_context_completions(is_function=True)) - elif symbol_names[-1] in ('trailer', 'dotted_name') and nodes[-1] == '.': - dot = self._module.get_leaf_for_position(self._position) - atom_expr = call_of_leaf(dot.get_previous_leaf()) - completion_names += self._trailer_completions(atom_expr) - else: - completion_names += self._global_completions() - completion_names += self._get_class_context_completions(is_function=False) - - if 'trailer' in symbol_names: - call_signatures = self._call_signatures_method() - completion_names += get_call_signature_param_names(call_signatures) - - return completion_names - - def _get_keyword_completion_names(self, keywords_): - for k in keywords_: - yield keywords.keyword(self._evaluator, k).name - - def _global_completions(self): - scope = get_user_scope(self._module, self._position) - if not scope.is_scope(): # Might be a flow (if/while/etc). - scope = scope.get_parent_scope() - scope = self._evaluator.wrap(scope) - debug.dbg('global completion scope: %s', scope) - names_dicts = global_names_dict_generator( - self._evaluator, - scope, - self._position - ) - completion_names = [] - for names_dict, pos in names_dicts: - names = list(chain.from_iterable(names_dict.values())) - if not names: - continue - completion_names += filter_definition_names( - names, self._module.get_statement_for_position(self._position), pos - ) - return completion_names - - def _trailer_completions(self, atom_expr): - scopes = self._evaluator.eval_element(atom_expr) - completion_names = [] - debug.dbg('trailer completion scopes: %s', scopes) - for s in scopes: - names = [] - for names_dict in s.names_dicts(search_global=False): - names += chain.from_iterable(names_dict.values()) - - completion_names += filter_definition_names( - names, self._module.get_statement_for_position(self._position) - ) - return completion_names - - def _parse_dotted_names(self, nodes): - level = 0 - names = [] - for node in nodes[1:]: - if node in ('.', '...'): - if not names: - level += len(node.value) - elif node.type == 'dotted_name': - names += node.children[::2] - elif node.type == 'name': - names.append(node) - else: - break - return level, names - - def _get_importer_names(self, names, level=0, only_modules=True): - names = [str(n) for n in names] - i = imports.Importer(self._evaluator, names, self._module, level) - return i.completion_names(self._evaluator, only_modules=only_modules) - - def _get_class_context_completions(self, is_function=True): - """ - Autocomplete inherited methods when overriding in child class. - """ - leaf = self._module.get_leaf_for_position(self._position, include_prefixes=True) - cls = leaf.get_parent_until(tree.Class) - if isinstance(cls, (tree.Class, tree.Function)): - # Complete the methods that are defined in the super classes. - cls = self._evaluator.wrap(cls) - else: - return - - if cls.start_pos[1] >= leaf.start_pos[1]: - return - - names_dicts = cls.names_dicts(search_global=False, is_instance=True) - # The first dict is the dictionary of class itself. - next(names_dicts) - for names_dict in names_dicts: - for values in names_dict.values(): - for value in values: - if (value.parent.type == 'funcdef') == is_function: - yield value diff --git a/pythonFiles/preview/jedi/api/helpers.py b/pythonFiles/preview/jedi/api/helpers.py deleted file mode 100644 index 8102cc7f2e23..000000000000 --- a/pythonFiles/preview/jedi/api/helpers.py +++ /dev/null @@ -1,315 +0,0 @@ -""" -Helpers for the API -""" -import re -from collections import namedtuple - -from jedi._compatibility import u -from jedi.evaluate.helpers import call_of_leaf -from jedi import parser -from jedi.parser import tokenize -from jedi.cache import time_cache -from jedi import common - - -CompletionParts = namedtuple('CompletionParts', ['path', 'has_dot', 'name']) - - -def sorted_definitions(defs): - # Note: `or ''` below is required because `module_path` could be - return sorted(defs, key=lambda x: (x.module_path or '', x.line or 0, x.column or 0)) - - -def get_on_completion_name(module, lines, position): - leaf = module.get_leaf_for_position(position) - if leaf is None or leaf.type in ('string', 'error_leaf'): - # Completions inside strings are a bit special, we need to parse the - # string. The same is true for comments and error_leafs. - line = lines[position[0] - 1] - # The first step of completions is to get the name - return re.search(r'(?!\d)\w+$|$', line[:position[1]]).group(0) - elif leaf.type not in ('name', 'keyword'): - return '' - - return leaf.value[:position[1] - leaf.start_pos[1]] - - -def _get_code(code_lines, start_pos, end_pos): - # Get relevant lines. - lines = code_lines[start_pos[0] - 1:end_pos[0]] - # Remove the parts at the end of the line. - lines[-1] = lines[-1][:end_pos[1]] - # Remove first line indentation. - lines[0] = lines[0][start_pos[1]:] - return '\n'.join(lines) - - -class OnErrorLeaf(Exception): - @property - def error_leaf(self): - return self.args[0] - - -def _is_on_comment(leaf, position): - # We might be on a comment. - if leaf.type == 'endmarker': - try: - dedent = leaf.get_previous_leaf() - if dedent.type == 'dedent' and dedent.prefix: - # TODO This is needed because the fast parser uses multiple - # endmarker tokens within a file which is obviously ugly. - # This is so ugly that I'm not even commenting how it exactly - # happens, but let me tell you that I want to get rid of it. - leaf = dedent - except IndexError: - pass - - comment_lines = common.splitlines(leaf.prefix) - difference = leaf.start_pos[0] - position[0] - prefix_start_pos = leaf.get_start_pos_of_prefix() - if difference == 0: - indent = leaf.start_pos[1] - elif position[0] == prefix_start_pos[0]: - indent = prefix_start_pos[1] - else: - indent = 0 - line = comment_lines[-difference - 1][:position[1] - indent] - return '#' in line - - -def _get_code_for_stack(code_lines, module, position): - leaf = module.get_leaf_for_position(position, include_prefixes=True) - # It might happen that we're on whitespace or on a comment. This means - # that we would not get the right leaf. - if leaf.start_pos >= position: - if _is_on_comment(leaf, position): - return u('') - - # If we're not on a comment simply get the previous leaf and proceed. - try: - leaf = leaf.get_previous_leaf() - except IndexError: - return u('') # At the beginning of the file. - - is_after_newline = leaf.type == 'newline' - while leaf.type == 'newline': - try: - leaf = leaf.get_previous_leaf() - except IndexError: - return u('') - - if leaf.type in ('indent', 'dedent'): - return u('') - elif leaf.type == 'error_leaf' or leaf.type == 'string': - # Error leafs cannot be parsed, completion in strings is also - # impossible. - raise OnErrorLeaf(leaf) - else: - if leaf == ';': - user_stmt = leaf.parent - else: - user_stmt = leaf.get_definition() - if user_stmt.parent.type == 'simple_stmt': - user_stmt = user_stmt.parent - - if is_after_newline: - if user_stmt.start_pos[1] > position[1]: - # This means that it's actually a dedent and that means that we - # start without context (part of a suite). - return u('') - - # This is basically getting the relevant lines. - return _get_code(code_lines, user_stmt.get_start_pos_of_prefix(), position) - - -def get_stack_at_position(grammar, code_lines, module, pos): - """ - Returns the possible node names (e.g. import_from, xor_test or yield_stmt). - """ - class EndMarkerReached(Exception): - pass - - def tokenize_without_endmarker(code): - tokens = tokenize.source_tokens(code, use_exact_op_types=True) - for token_ in tokens: - if token_.string == safeword: - raise EndMarkerReached() - else: - yield token_ - - code = _get_code_for_stack(code_lines, module, pos) - # We use a word to tell Jedi when we have reached the start of the - # completion. - # Use Z as a prefix because it's not part of a number suffix. - safeword = 'ZZZ_USER_WANTS_TO_COMPLETE_HERE_WITH_JEDI' - # Remove as many indents from **all** code lines as possible. - code = code + safeword - - p = parser.ParserWithRecovery(grammar, code, start_parsing=False) - try: - p.parse(tokenizer=tokenize_without_endmarker(code)) - except EndMarkerReached: - return Stack(p.stack) - raise SystemError("This really shouldn't happen. There's a bug in Jedi.") - - -class Stack(list): - def get_node_names(self, grammar): - for dfa, state, (node_number, nodes) in self: - yield grammar.number2symbol[node_number] - - def get_nodes(self): - for dfa, state, (node_number, nodes) in self: - for node in nodes: - yield node - - -def get_possible_completion_types(grammar, stack): - def add_results(label_index): - try: - grammar_labels.append(inversed_tokens[label_index]) - except KeyError: - try: - keywords.append(inversed_keywords[label_index]) - except KeyError: - t, v = grammar.labels[label_index] - assert t >= 256 - # See if it's a symbol and if we're in its first set - inversed_keywords - itsdfa = grammar.dfas[t] - itsstates, itsfirst = itsdfa - for first_label_index in itsfirst.keys(): - add_results(first_label_index) - - inversed_keywords = dict((v, k) for k, v in grammar.keywords.items()) - inversed_tokens = dict((v, k) for k, v in grammar.tokens.items()) - - keywords = [] - grammar_labels = [] - - def scan_stack(index): - dfa, state, node = stack[index] - states, first = dfa - arcs = states[state] - - for label_index, new_state in arcs: - if label_index == 0: - # An accepting state, check the stack below. - scan_stack(index - 1) - else: - add_results(label_index) - - scan_stack(-1) - - return keywords, grammar_labels - - -def evaluate_goto_definition(evaluator, leaf): - if leaf.type == 'name': - # In case of a name we can just use goto_definition which does all the - # magic itself. - return evaluator.goto_definitions(leaf) - - node = None - parent = leaf.parent - if parent.type == 'atom': - node = leaf.parent - elif parent.type == 'trailer': - node = call_of_leaf(leaf) - - if node is None: - return [] - return evaluator.eval_element(node) - - -CallSignatureDetails = namedtuple( - 'CallSignatureDetails', - ['bracket_leaf', 'call_index', 'keyword_name_str'] -) - - -def _get_index_and_key(nodes, position): - """ - Returns the amount of commas and the keyword argument string. - """ - nodes_before = [c for c in nodes if c.start_pos < position] - if nodes_before[-1].type == 'arglist': - nodes_before = [c for c in nodes_before[-1].children if c.start_pos < position] - - key_str = None - - if nodes_before: - last = nodes_before[-1] - if last.type == 'argument' and last.children[1].end_pos <= position: - # Checked if the argument - key_str = last.children[0].value - elif last == '=': - key_str = nodes_before[-2].value - - return nodes_before.count(','), key_str - - -def _get_call_signature_details_from_error_node(node, position): - for index, element in reversed(list(enumerate(node.children))): - # `index > 0` means that it's a trailer and not an atom. - if element == '(' and element.end_pos <= position and index > 0: - # It's an error node, we don't want to match too much, just - # until the parentheses is enough. - children = node.children[index:] - name = element.get_previous_leaf() - if name.type == 'name' or name.parent.type in ('trailer', 'atom'): - return CallSignatureDetails( - element, - *_get_index_and_key(children, position) - ) - - -def get_call_signature_details(module, position): - leaf = module.get_leaf_for_position(position, include_prefixes=True) - if leaf == ')': - if leaf.end_pos == position: - leaf = leaf.get_next_leaf() - # Now that we know where we are in the syntax tree, we start to look at - # parents for possible function definitions. - node = leaf.parent - while node is not None: - if node.type in ('funcdef', 'classdef'): - # Don't show call signatures if there's stuff before it that just - # makes it feel strange to have a call signature. - return None - - for n in node.children[::-1]: - if n.start_pos < position and n.type == 'error_node': - result = _get_call_signature_details_from_error_node(n, position) - if result is not None: - return result - - if node.type == 'trailer' and node.children[0] == '(': - leaf = node.get_previous_leaf() - return CallSignatureDetails( - node.children[0], *_get_index_and_key(node.children, position)) - - node = node.parent - - return None - - -@time_cache("call_signatures_validity") -def cache_call_signatures(evaluator, bracket_leaf, code_lines, user_pos): - """This function calculates the cache key.""" - index = user_pos[0] - 1 - - before_cursor = code_lines[index][:user_pos[1]] - other_lines = code_lines[bracket_leaf.start_pos[0]:index] - whole = '\n'.join(other_lines + [before_cursor]) - before_bracket = re.match(r'.*\(', whole, re.DOTALL) - - module_path = bracket_leaf.get_parent_until().path - if module_path is None: - yield None # Don't cache! - else: - yield (module_path, before_bracket, bracket_leaf.start_pos) - yield evaluate_goto_definition( - evaluator, - bracket_leaf.get_previous_leaf() - ) diff --git a/pythonFiles/preview/jedi/api/interpreter.py b/pythonFiles/preview/jedi/api/interpreter.py deleted file mode 100644 index a6778a6cd65a..000000000000 --- a/pythonFiles/preview/jedi/api/interpreter.py +++ /dev/null @@ -1,67 +0,0 @@ -""" -TODO Some parts of this module are still not well documented. -""" -import copy - -from jedi.cache import underscore_memoization -from jedi.evaluate import helpers -from jedi.evaluate.representation import ModuleWrapper -from jedi.evaluate.compiled import mixed - - -class MixedModule(object): - resets_positions = True - type = 'mixed_module' - - def __init__(self, evaluator, parser_module, namespaces): - self._evaluator = evaluator - self._namespaces = namespaces - - self._namespace_objects = [type('jedi_namespace', (), n) for n in namespaces] - self._wrapped_module = ModuleWrapper(evaluator, parser_module) - # Usually we are dealing with very small code sizes when it comes to - # interpreter modules. In this case we just copy the whole syntax tree - # to be able to modify it. - self._parser_module = copy.deepcopy(parser_module) - - for child in self._parser_module.children: - child.parent = self - - def names_dicts(self, search_global): - for names_dict in self._wrapped_module.names_dicts(search_global): - yield names_dict - - for namespace_obj in self._namespace_objects: - m = mixed.MixedObject(self._evaluator, namespace_obj, self._parser_module.name) - for names_dict in m.names_dicts(False): - yield names_dict - - def __getattr__(self, name): - return getattr(self._parser_module, name) - - -class LazyName(helpers.FakeName): - def __init__(self, evaluator, module, name, value): - super(LazyName, self).__init__(name) - self._module = module - self._evaluator = evaluator - self._value = value - self._name = name - - def is_definition(self): - return True - - @property - @underscore_memoization - def parent(self): - """ - Creating fake statements for the interpreter. - - Here we are trying to link back to Python code, if possible. This means - we try to find the python module for a name (not the builtin). - """ - return mixed.create(self._evaluator, self._value) - - @parent.setter - def parent(self, value): - """Needed because the super class tries to set parent.""" diff --git a/pythonFiles/preview/jedi/api/keywords.py b/pythonFiles/preview/jedi/api/keywords.py deleted file mode 100644 index d6a72b1db98e..000000000000 --- a/pythonFiles/preview/jedi/api/keywords.py +++ /dev/null @@ -1,132 +0,0 @@ -import pydoc -import keyword - -from jedi._compatibility import is_py3, is_py35 -from jedi import common -from jedi.evaluate.helpers import FakeName -from jedi.parser.tree import Leaf -try: - from pydoc_data import topics as pydoc_topics -except ImportError: - # Python 2 - try: - import pydoc_topics - except ImportError: - # This is for Python 3 embeddable version, which dont have - # pydoc_data module in its file python3x.zip. - pydoc_topics = None - -if is_py3: - if is_py35: - # in python 3.5 async and await are not proper keywords, but for - # completion pursposes should as as though they are - keys = keyword.kwlist + ["async", "await"] - else: - keys = keyword.kwlist -else: - keys = keyword.kwlist + ['None', 'False', 'True'] - - -def has_inappropriate_leaf_keyword(pos, module): - relevant_errors = filter( - lambda error: error.first_pos[0] == pos[0], - module.error_statement_stacks) - - for error in relevant_errors: - if error.next_token in keys: - return True - - return False - - -def completion_names(evaluator, stmt, pos, module): - keyword_list = all_keywords(evaluator) - - if not isinstance(stmt, Leaf) or has_inappropriate_leaf_keyword(pos, module): - keyword_list = filter( - lambda keyword: not keyword.only_valid_as_leaf, - keyword_list - ) - return [keyword.name for keyword in keyword_list] - - -def all_keywords(evaluator, pos=(0, 0)): - return set([Keyword(evaluator, k, pos) for k in keys]) - - -def keyword(evaluator, string, pos=(0, 0)): - if string in keys: - return Keyword(evaluator, string, pos) - else: - return None - - -def get_operator(evaluator, string, pos): - return Keyword(evaluator, string, pos) - - -keywords_only_valid_as_leaf = ( - 'continue', - 'break', -) - - -class Keyword(object): - type = 'completion_keyword' - - def __init__(self, evaluator, name, pos): - self.name = FakeName(name, self, pos) - self.start_pos = pos - self.parent = evaluator.BUILTINS - - def get_parent_until(self): - return self.parent - - @property - def only_valid_as_leaf(self): - return self.name.value in keywords_only_valid_as_leaf - - @property - def names(self): - """ For a `parsing.Name` like comparision """ - return [self.name] - - @property - def docstr(self): - return imitate_pydoc(self.name) - - def __repr__(self): - return '<%s: %s>' % (type(self).__name__, self.name) - - -def imitate_pydoc(string): - """ - It's not possible to get the pydoc's without starting the annoying pager - stuff. - """ - if pydoc_topics is None: - return '' - - # str needed because of possible unicode stuff in py2k (pydoc doesn't work - # with unicode strings) - string = str(string) - h = pydoc.help - with common.ignored(KeyError): - # try to access symbols - string = h.symbols[string] - string, _, related = string.partition(' ') - - get_target = lambda s: h.topics.get(s, h.keywords.get(s)) - while isinstance(string, str): - string = get_target(string) - - try: - # is a tuple now - label, related = string - except TypeError: - return '' - - try: - return pydoc_topics.topics[label] if pydoc_topics else '' - except KeyError: - return '' diff --git a/pythonFiles/preview/jedi/api/usages.py b/pythonFiles/preview/jedi/api/usages.py deleted file mode 100644 index ecb885639032..000000000000 --- a/pythonFiles/preview/jedi/api/usages.py +++ /dev/null @@ -1,49 +0,0 @@ -from jedi._compatibility import unicode -from jedi.api import classes -from jedi.parser import tree -from jedi.evaluate import imports - - -def usages(evaluator, definition_names, mods): - """ - :param definitions: list of Name - """ - def compare_array(definitions): - """ `definitions` are being compared by module/start_pos, because - sometimes the id's of the objects change (e.g. executions). - """ - result = [] - for d in definitions: - module = d.get_parent_until() - result.append((module, d.start_pos)) - return result - - search_name = unicode(list(definition_names)[0]) - compare_definitions = compare_array(definition_names) - mods |= set([d.get_parent_until() for d in definition_names]) - definitions = [] - for m in imports.get_modules_containing_name(evaluator, mods, search_name): - try: - check_names = m.used_names[search_name] - except KeyError: - continue - for name in check_names: - - result = evaluator.goto(name) - if [c for c in compare_array(result) if c in compare_definitions]: - definitions.append(classes.Definition(evaluator, name)) - # Previous definitions might be imports, so include them - # (because goto might return that import name). - compare_definitions += compare_array([name]) - return definitions - - -def usages_add_import_modules(evaluator, definitions): - """ Adds the modules of the imports """ - new = set() - for d in definitions: - imp_or_stmt = d.get_definition() - if isinstance(imp_or_stmt, tree.Import): - s = imports.ImportWrapper(evaluator, d) - new |= set(s.follow(is_goto=True)) - return set(definitions) | new diff --git a/pythonFiles/preview/jedi/cache.py b/pythonFiles/preview/jedi/cache.py deleted file mode 100644 index 8dc82544a87b..000000000000 --- a/pythonFiles/preview/jedi/cache.py +++ /dev/null @@ -1,127 +0,0 @@ -""" -This caching is very important for speed and memory optimizations. There's -nothing really spectacular, just some decorators. The following cache types are -available: - -- module caching (`load_parser` and `save_parser`), which uses pickle and is - really important to assure low load times of modules like ``numpy``. -- ``time_cache`` can be used to cache something for just a limited time span, - which can be useful if there's user interaction and the user cannot react - faster than a certain time. - -This module is one of the reasons why |jedi| is not thread-safe. As you can see -there are global variables, which are holding the cache information. Some of -these variables are being cleaned after every API usage. -""" -import time - -from jedi import settings -from jedi.parser.utils import parser_cache -from jedi.parser.utils import underscore_memoization - -_time_caches = {} - - -def clear_time_caches(delete_all=False): - """ Jedi caches many things, that should be completed after each completion - finishes. - - :param delete_all: Deletes also the cache that is normally not deleted, - like parser cache, which is important for faster parsing. - """ - global _time_caches - - if delete_all: - for cache in _time_caches.values(): - cache.clear() - parser_cache.clear() - else: - # normally just kill the expired entries, not all - for tc in _time_caches.values(): - # check time_cache for expired entries - for key, (t, value) in list(tc.items()): - if t < time.time(): - # delete expired entries - del tc[key] - - -def time_cache(time_add_setting): - """ - This decorator works as follows: Call it with a setting and after that - use the function with a callable that returns the key. - But: This function is only called if the key is not available. After a - certain amount of time (`time_add_setting`) the cache is invalid. - - If the given key is None, the function will not be cached. - """ - def _temp(key_func): - dct = {} - _time_caches[time_add_setting] = dct - - def wrapper(*args, **kwargs): - generator = key_func(*args, **kwargs) - key = next(generator) - try: - expiry, value = dct[key] - if expiry > time.time(): - return value - except KeyError: - pass - - value = next(generator) - time_add = getattr(settings, time_add_setting) - if key is not None: - dct[key] = time.time() + time_add, value - return value - return wrapper - return _temp - - -def memoize_method(method): - """A normal memoize function.""" - def wrapper(self, *args, **kwargs): - dct = self.__dict__.setdefault('_memoize_method_dct', {}) - key = (args, frozenset(kwargs.items())) - try: - return dct[key] - except KeyError: - result = method(self, *args, **kwargs) - dct[key] = result - return result - return wrapper - - -def cache_star_import(func): - @time_cache("star_import_cache_validity") - def wrapper(self): - yield self.base # The cache key - yield func(self) - return wrapper - - -def _invalidate_star_import_cache_module(module, only_main=False): - """ Important if some new modules are being reparsed """ - try: - t, modules = _time_caches['star_import_cache_validity'][module] - except KeyError: - pass - else: - del _time_caches['star_import_cache_validity'][module] - - # This stuff was part of load_parser. However since we're most likely - # not going to use star import caching anymore, just ignore it. - #else: - # In case there is already a module cached and this module - # has to be reparsed, we also need to invalidate the import - # caches. - # _invalidate_star_import_cache_module(parser_cache_item.parser.module) - - -def invalidate_star_import_cache(path): - """On success returns True.""" - try: - parser_cache_item = parser_cache[path] - except KeyError: - pass - else: - _invalidate_star_import_cache_module(parser_cache_item.parser.module) diff --git a/pythonFiles/preview/jedi/common.py b/pythonFiles/preview/jedi/common.py deleted file mode 100644 index ae10f423f393..000000000000 --- a/pythonFiles/preview/jedi/common.py +++ /dev/null @@ -1,169 +0,0 @@ -""" A universal module with functions / classes without dependencies. """ -import sys -import contextlib -import functools -import re -from itertools import chain -from ast import literal_eval - -from jedi._compatibility import unicode, reraise -from jedi import settings - - -class UncaughtAttributeError(Exception): - """ - Important, because `__getattr__` and `hasattr` catch AttributeErrors - implicitly. This is really evil (mainly because of `__getattr__`). - `hasattr` in Python 2 is even more evil, because it catches ALL exceptions. - Therefore this class originally had to be derived from `BaseException` - instead of `Exception`. But because I removed relevant `hasattr` from - the code base, we can now switch back to `Exception`. - - :param base: return values of sys.exc_info(). - """ - - -def safe_property(func): - return property(reraise_uncaught(func)) - - -def reraise_uncaught(func): - """ - Re-throw uncaught `AttributeError`. - - Usage: Put ``@rethrow_uncaught`` in front of the function - which does **not** suppose to raise `AttributeError`. - - AttributeError is easily get caught by `hasattr` and another - ``except AttributeError`` clause. This becomes problem when you use - a lot of "dynamic" attributes (e.g., using ``@property``) because you - can't distinguish if the property does not exist for real or some code - inside of the "dynamic" attribute through that error. In a well - written code, such error should not exist but getting there is very - difficult. This decorator is to help us getting there by changing - `AttributeError` to `UncaughtAttributeError` to avoid unexpected catch. - This helps us noticing bugs earlier and facilitates debugging. - - .. note:: Treating StopIteration here is easy. - Add that feature when needed. - """ - @functools.wraps(func) - def wrapper(*args, **kwds): - try: - return func(*args, **kwds) - except AttributeError: - exc_info = sys.exc_info() - reraise(UncaughtAttributeError(exc_info[1]), exc_info[2]) - return wrapper - - -class PushBackIterator(object): - def __init__(self, iterator): - self.pushes = [] - self.iterator = iterator - self.current = None - - def push_back(self, value): - self.pushes.append(value) - - def __iter__(self): - return self - - def next(self): - """ Python 2 Compatibility """ - return self.__next__() - - def __next__(self): - if self.pushes: - self.current = self.pushes.pop() - else: - self.current = next(self.iterator) - return self.current - - -@contextlib.contextmanager -def scale_speed_settings(factor): - a = settings.max_executions - b = settings.max_until_execution_unique - settings.max_executions *= factor - settings.max_until_execution_unique *= factor - try: - yield - finally: - settings.max_executions = a - settings.max_until_execution_unique = b - - -def indent_block(text, indention=' '): - """This function indents a text block with a default of four spaces.""" - temp = '' - while text and text[-1] == '\n': - temp += text[-1] - text = text[:-1] - lines = text.split('\n') - return '\n'.join(map(lambda s: indention + s, lines)) + temp - - -@contextlib.contextmanager -def ignored(*exceptions): - """ - Context manager that ignores all of the specified exceptions. This will - be in the standard library starting with Python 3.4. - """ - try: - yield - except exceptions: - pass - - -def source_to_unicode(source, encoding=None): - def detect_encoding(): - """ - For the implementation of encoding definitions in Python, look at: - - http://www.python.org/dev/peps/pep-0263/ - - http://docs.python.org/2/reference/lexical_analysis.html#encoding-declarations - """ - byte_mark = literal_eval(r"b'\xef\xbb\xbf'") - if source.startswith(byte_mark): - # UTF-8 byte-order mark - return 'utf-8' - - first_two_lines = re.match(br'(?:[^\n]*\n){0,2}', source).group(0) - possible_encoding = re.search(br"coding[=:]\s*([-\w.]+)", - first_two_lines) - if possible_encoding: - return possible_encoding.group(1) - else: - # the default if nothing else has been set -> PEP 263 - return encoding if encoding is not None else 'utf-8' - - if isinstance(source, unicode): - # only cast str/bytes - return source - - encoding = detect_encoding() - if not isinstance(encoding, unicode): - encoding = unicode(encoding, 'utf-8', 'replace') - # cast to unicode by default - return unicode(source, encoding, 'replace') - - -def splitlines(string): - """ - A splitlines for Python code. In contrast to Python's ``str.splitlines``, - looks at form feeds and other special characters as normal text. Just - splits ``\n`` and ``\r\n``. - Also different: Returns ``['']`` for an empty string input. - """ - return re.split('\n|\r\n', string) - - -def unite(iterable): - """Turns a two dimensional array into a one dimensional.""" - return set(chain.from_iterable(iterable)) - - -def to_list(func): - def wrapper(*args, **kwargs): - return list(func(*args, **kwargs)) - return wrapper diff --git a/pythonFiles/preview/jedi/debug.py b/pythonFiles/preview/jedi/debug.py deleted file mode 100644 index 692f9c82842b..000000000000 --- a/pythonFiles/preview/jedi/debug.py +++ /dev/null @@ -1,133 +0,0 @@ -from jedi._compatibility import encoding, is_py3, u -import inspect -import os -import time - -def _lazy_colorama_init(): - """ - Lazily init colorama if necessary, not to screw up stdout is debug not - enabled. - - This version of the function does nothing. - """ - pass - -_inited=False - -try: - if os.name == 'nt': - # Does not work on Windows, as pyreadline and colorama interfere - raise ImportError - else: - # Use colorama for nicer console output. - from colorama import Fore, init - from colorama import initialise - def _lazy_colorama_init(): - """ - Lazily init colorama if necessary, not to screw up stdout is - debug not enabled. - - This version of the function does init colorama. - """ - global _inited - if not _inited: - # pytest resets the stream at the end - causes troubles. Since - # after every output the stream is reset automatically we don't - # need this. - initialise.atexit_done = True - try: - init() - except Exception: - # Colorama fails with initializing under vim and is buggy in - # version 0.3.6. - pass - _inited = True - -except ImportError: - class Fore(object): - RED = '' - GREEN = '' - YELLOW = '' - MAGENTA = '' - RESET = '' - -NOTICE = object() -WARNING = object() -SPEED = object() - -enable_speed = False -enable_warning = False -enable_notice = False - -# callback, interface: level, str -debug_function = None -ignored_modules = ['jedi.parser'] -_debug_indent = 0 -_start_time = time.time() - - -def reset_time(): - global _start_time, _debug_indent - _start_time = time.time() - _debug_indent = 0 - - -def increase_indent(func): - """Decorator for makin """ - def wrapper(*args, **kwargs): - global _debug_indent - _debug_indent += 1 - try: - return func(*args, **kwargs) - finally: - _debug_indent -= 1 - return wrapper - - -def dbg(message, *args, **kwargs): - """ Looks at the stack, to see if a debug message should be printed. """ - # Python 2 compatibility, because it doesn't understand default args - color = kwargs.pop('color', 'GREEN') - assert color - - if debug_function and enable_notice: - frm = inspect.stack()[1] - mod = inspect.getmodule(frm[0]) - if not (mod.__name__ in ignored_modules): - i = ' ' * _debug_indent - _lazy_colorama_init() - debug_function(color, i + 'dbg: ' + message % tuple(u(repr(a)) for a in args)) - - -def warning(message, *args, **kwargs): - format = kwargs.pop('format', True) - assert not kwargs - - if debug_function and enable_warning: - i = ' ' * _debug_indent - if format: - message = message % tuple(u(repr(a)) for a in args) - debug_function('RED', i + 'warning: ' + message) - - -def speed(name): - if debug_function and enable_speed: - now = time.time() - i = ' ' * _debug_indent - debug_function('YELLOW', i + 'speed: ' + '%s %s' % (name, now - _start_time)) - - -def print_to_stdout(color, str_out): - """ - The default debug function that prints to standard out. - - :param str color: A string that is an attribute of ``colorama.Fore``. - """ - col = getattr(Fore, color) - _lazy_colorama_init() - if not is_py3: - str_out = str_out.encode(encoding, 'replace') - print(col + str_out + Fore.RESET) - - -# debug_function = print_to_stdout diff --git a/pythonFiles/preview/jedi/evaluate/__init__.py b/pythonFiles/preview/jedi/evaluate/__init__.py deleted file mode 100644 index fe2c70237afd..000000000000 --- a/pythonFiles/preview/jedi/evaluate/__init__.py +++ /dev/null @@ -1,536 +0,0 @@ -""" -Evaluation of Python code in |jedi| is based on three assumptions: - -* The code uses as least side effects as possible. Jedi understands certain - list/tuple/set modifications, but there's no guarantee that Jedi detects - everything (list.append in different modules for example). -* No magic is being used: - - - metaclasses - - ``setattr()`` / ``__import__()`` - - writing to ``globals()``, ``locals()``, ``object.__dict__`` -* The programmer is not a total dick, e.g. like `this - `_ :-) - -The actual algorithm is based on a principle called lazy evaluation. If you -don't know about it, google it. That said, the typical entry point for static -analysis is calling ``eval_statement``. There's separate logic for -autocompletion in the API, the evaluator is all about evaluating an expression. - -Now you need to understand what follows after ``eval_statement``. Let's -make an example:: - - import datetime - datetime.date.toda# <-- cursor here - -First of all, this module doesn't care about completion. It really just cares -about ``datetime.date``. At the end of the procedure ``eval_statement`` will -return the ``date`` class. - -To *visualize* this (simplified): - -- ``Evaluator.eval_statement`` doesn't do much, because there's no assignment. -- ``Evaluator.eval_element`` cares for resolving the dotted path -- ``Evaluator.find_types`` searches for global definitions of datetime, which - it finds in the definition of an import, by scanning the syntax tree. -- Using the import logic, the datetime module is found. -- Now ``find_types`` is called again by ``eval_element`` to find ``date`` - inside the datetime module. - -Now what would happen if we wanted ``datetime.date.foo.bar``? Two more -calls to ``find_types``. However the second call would be ignored, because the -first one would return nothing (there's no foo attribute in ``date``). - -What if the import would contain another ``ExprStmt`` like this:: - - from foo import bar - Date = bar.baz - -Well... You get it. Just another ``eval_statement`` recursion. It's really -easy. Python can obviously get way more complicated then this. To understand -tuple assignments, list comprehensions and everything else, a lot more code had -to be written. - -Jedi has been tested very well, so you can just start modifying code. It's best -to write your own test first for your "new" feature. Don't be scared of -breaking stuff. As long as the tests pass, you're most likely to be fine. - -I need to mention now that lazy evaluation is really good because it -only *evaluates* what needs to be *evaluated*. All the statements and modules -that are not used are just being ignored. -""" - -import copy -import sys -from itertools import chain - -from jedi.parser import tree -from jedi import debug -from jedi.evaluate import representation as er -from jedi.evaluate import imports -from jedi.evaluate import recursion -from jedi.evaluate import iterable -from jedi.evaluate.cache import memoize_default -from jedi.evaluate import stdlib -from jedi.evaluate import finder -from jedi.evaluate import compiled -from jedi.evaluate import precedence -from jedi.evaluate import param -from jedi.evaluate import helpers - - -class Evaluator(object): - def __init__(self, grammar, sys_path=None): - self.grammar = grammar - self.memoize_cache = {} # for memoize decorators - # To memorize modules -> equals `sys.modules`. - self.modules = {} # like `sys.modules`. - self.compiled_cache = {} # see `evaluate.compiled.create()` - self.mixed_cache = {} # see `evaluate.compiled.mixed.create()` - self.analysis = [] - self.predefined_if_name_dict_dict = {} - self.dynamic_params_depth = 0 - self.is_analysis = False - - if sys_path is None: - sys_path = sys.path - self.sys_path = copy.copy(sys_path) - try: - self.sys_path.remove('') - except ValueError: - pass - - self.reset_recursion_limitations() - - # Constants - self.BUILTINS = compiled.get_special_object(self, 'BUILTINS') - - def reset_recursion_limitations(self): - self.recursion_detector = recursion.RecursionDetector(self) - self.execution_recursion_detector = recursion.ExecutionRecursionDetector(self) - - def wrap(self, element): - if isinstance(element, (er.Wrapper, er.InstanceElement, - er.ModuleWrapper, er.FunctionExecution, er.Instance, compiled.CompiledObject)) or element is None: - # TODO this is so ugly, please refactor. - return element - - if element.type == 'classdef': - return er.Class(self, element) - elif element.type == 'funcdef': - return er.Function(self, element) - elif element.type == 'lambda': - return er.LambdaWrapper(self, element) - elif element.type == 'file_input': - return er.ModuleWrapper(self, element) - else: - return element - - def find_types(self, scope, name_str, position=None, search_global=False, - is_goto=False): - """ - This is the search function. The most important part to debug. - `remove_statements` and `filter_statements` really are the core part of - this completion. - - :param position: Position of the last statement -> tuple of line, column - :return: List of Names. Their parents are the types. - """ - f = finder.NameFinder(self, scope, name_str, position) - scopes = f.scopes(search_global) - if is_goto: - return f.filter_name(scopes) - return f.find(scopes, attribute_lookup=not search_global) - - #@memoize_default(default=[], evaluator_is_first_arg=True) - #@recursion.recursion_decorator - @debug.increase_indent - def eval_statement(self, stmt, seek_name=None): - """ - The starting point of the completion. A statement always owns a call - list, which are the calls, that a statement does. In case multiple - names are defined in the statement, `seek_name` returns the result for - this name. - - :param stmt: A `tree.ExprStmt`. - """ - debug.dbg('eval_statement %s (%s)', stmt, seek_name) - rhs = stmt.get_rhs() - types = self.eval_element(rhs) - - if seek_name: - types = finder.check_tuple_assignments(self, types, seek_name) - - first_operation = stmt.first_operation() - if first_operation not in ('=', None) and not isinstance(stmt, er.InstanceElement) and first_operation.type == 'operator': # TODO don't check for this. - # `=` is always the last character in aug assignments -> -1 - operator = copy.copy(first_operation) - operator.value = operator.value[:-1] - name = str(stmt.get_defined_names()[0]) - parent = self.wrap(stmt.get_parent_scope()) - left = self.find_types(parent, name, stmt.start_pos, search_global=True) - - for_stmt = stmt.get_parent_until(tree.ForStmt) - if isinstance(for_stmt, tree.ForStmt) and types \ - and for_stmt.defines_one_name(): - # Iterate through result and add the values, that's possible - # only in for loops without clutter, because they are - # predictable. Also only do it, if the variable is not a tuple. - node = for_stmt.get_input_node() - for_iterables = self.eval_element(node) - ordered = list(iterable.py__iter__(self, for_iterables, node)) - - for index_types in ordered: - dct = {str(for_stmt.children[1]): index_types} - self.predefined_if_name_dict_dict[for_stmt] = dct - t = self.eval_element(rhs) - left = precedence.calculate(self, left, operator, t) - types = left - if ordered: - # If there are no for entries, we cannot iterate and the - # types are defined by += entries. Therefore the for loop - # is never called. - del self.predefined_if_name_dict_dict[for_stmt] - else: - types = precedence.calculate(self, left, operator, types) - debug.dbg('eval_statement result %s', types) - return types - - def eval_element(self, element): - if isinstance(element, iterable.AlreadyEvaluated): - return set(element) - elif isinstance(element, iterable.MergedNodes): - return iterable.unite(self.eval_element(e) for e in element) - - if_stmt = element.get_parent_until((tree.IfStmt, tree.ForStmt, tree.IsScope)) - predefined_if_name_dict = self.predefined_if_name_dict_dict.get(if_stmt) - if predefined_if_name_dict is None and isinstance(if_stmt, tree.IfStmt): - if_stmt_test = if_stmt.children[1] - name_dicts = [{}] - # If we already did a check, we don't want to do it again -> If - # predefined_if_name_dict_dict is filled, we stop. - # We don't want to check the if stmt itself, it's just about - # the content. - if element.start_pos > if_stmt_test.end_pos: - # Now we need to check if the names in the if_stmt match the - # names in the suite. - if_names = helpers.get_names_of_node(if_stmt_test) - element_names = helpers.get_names_of_node(element) - str_element_names = [str(e) for e in element_names] - if any(str(i) in str_element_names for i in if_names): - for if_name in if_names: - definitions = self.goto_definitions(if_name) - # Every name that has multiple different definitions - # causes the complexity to rise. The complexity should - # never fall below 1. - if len(definitions) > 1: - if len(name_dicts) * len(definitions) > 16: - debug.dbg('Too many options for if branch evaluation %s.', if_stmt) - # There's only a certain amount of branches - # Jedi can evaluate, otherwise it will take to - # long. - name_dicts = [{}] - break - - original_name_dicts = list(name_dicts) - name_dicts = [] - for definition in definitions: - new_name_dicts = list(original_name_dicts) - for i, name_dict in enumerate(new_name_dicts): - new_name_dicts[i] = name_dict.copy() - new_name_dicts[i][str(if_name)] = [definition] - - name_dicts += new_name_dicts - else: - for name_dict in name_dicts: - name_dict[str(if_name)] = definitions - if len(name_dicts) > 1: - result = set() - for name_dict in name_dicts: - self.predefined_if_name_dict_dict[if_stmt] = name_dict - try: - result |= self._eval_element_not_cached(element) - finally: - del self.predefined_if_name_dict_dict[if_stmt] - return result - else: - return self._eval_element_if_evaluated(element) - return self._eval_element_cached(element) - else: - if predefined_if_name_dict: - return self._eval_element_not_cached(element) - else: - return self._eval_element_if_evaluated(element) - return self._eval_element_cached(element) - - def _eval_element_if_evaluated(self, element): - """ - TODO This function is temporary: Merge with eval_element. - """ - parent = element - while parent is not None: - parent = parent.parent - predefined_if_name_dict = self.predefined_if_name_dict_dict.get(parent) - if predefined_if_name_dict is not None: - return self._eval_element_not_cached(element) - return self._eval_element_cached(element) - - @memoize_default(default=set(), evaluator_is_first_arg=True) - def _eval_element_cached(self, element): - return self._eval_element_not_cached(element) - - @debug.increase_indent - def _eval_element_not_cached(self, element): - debug.dbg('eval_element %s@%s', element, element.start_pos) - types = set() - if isinstance(element, (tree.Name, tree.Literal)) or tree.is_node(element, 'atom'): - types = self._eval_atom(element) - elif isinstance(element, tree.Keyword): - # For False/True/None - if element.value in ('False', 'True', 'None'): - types.add(compiled.builtin_from_name(self, element.value)) - # else: print e.g. could be evaluated like this in Python 2.7 - elif element.isinstance(tree.Lambda): - types = set([er.LambdaWrapper(self, element)]) - elif element.isinstance(er.LambdaWrapper): - types = set([element]) # TODO this is no real evaluation. - elif element.type == 'expr_stmt': - types = self.eval_statement(element) - elif element.type in ('power', 'atom_expr'): - types = self._eval_atom(element.children[0]) - for trailer in element.children[1:]: - if trailer == '**': # has a power operation. - right = self.eval_element(element.children[2]) - types = set(precedence.calculate(self, types, trailer, right)) - break - types = self.eval_trailer(types, trailer) - elif element.type in ('testlist_star_expr', 'testlist',): - # The implicit tuple in statements. - types = set([iterable.ImplicitTuple(self, element)]) - elif element.type in ('not_test', 'factor'): - types = self.eval_element(element.children[-1]) - for operator in element.children[:-1]: - types = set(precedence.factor_calculate(self, types, operator)) - elif element.type == 'test': - # `x if foo else y` case. - types = (self.eval_element(element.children[0]) | - self.eval_element(element.children[-1])) - elif element.type == 'operator': - # Must be an ellipsis, other operators are not evaluated. - assert element.value == '...' - types = set([compiled.create(self, Ellipsis)]) - elif element.type == 'dotted_name': - types = self._eval_atom(element.children[0]) - for next_name in element.children[2::2]: - types = set(chain.from_iterable(self.find_types(typ, next_name) - for typ in types)) - types = types - elif element.type == 'eval_input': - types = self._eval_element_not_cached(element.children[0]) - elif element.type == 'annassign': - types = self.eval_element(element.children[1]) - else: - types = precedence.calculate_children(self, element.children) - debug.dbg('eval_element result %s', types) - return types - - def _eval_atom(self, atom): - """ - Basically to process ``atom`` nodes. The parser sometimes doesn't - generate the node (because it has just one child). In that case an atom - might be a name or a literal as well. - """ - if isinstance(atom, tree.Name): - # This is the first global lookup. - stmt = atom.get_definition() - scope = stmt.get_parent_until(tree.IsScope, include_current=True) - if isinstance(scope, (tree.Function, er.FunctionExecution)): - # Adjust scope: If the name is not in the suite, it's a param - # default or annotation and will be resolved as part of the - # parent scope. - colon = scope.children.index(':') - if atom.start_pos < scope.children[colon + 1].start_pos: - scope = scope.get_parent_scope() - if isinstance(stmt, tree.CompFor): - stmt = stmt.get_parent_until((tree.ClassOrFunc, tree.ExprStmt)) - if stmt.type != 'expr_stmt': - # We only need to adjust the start_pos for statements, because - # there the name cannot be used. - stmt = atom - return self.find_types(scope, atom, stmt.start_pos, search_global=True) - elif isinstance(atom, tree.Literal): - return set([compiled.create(self, atom.eval())]) - else: - c = atom.children - if c[0].type == 'string': - # Will be one string. - types = self._eval_atom(c[0]) - for string in c[1:]: - right = self._eval_atom(string) - types = precedence.calculate(self, types, '+', right) - return types - # Parentheses without commas are not tuples. - elif c[0] == '(' and not len(c) == 2 \ - and not(tree.is_node(c[1], 'testlist_comp') - and len(c[1].children) > 1): - return self.eval_element(c[1]) - - try: - comp_for = c[1].children[1] - except (IndexError, AttributeError): - pass - else: - if comp_for == ':': - # Dict comprehensions have a colon at the 3rd index. - try: - comp_for = c[1].children[3] - except IndexError: - pass - - if comp_for.type == 'comp_for': - return set([iterable.Comprehension.from_atom(self, atom)]) - return set([iterable.Array(self, atom)]) - - def eval_trailer(self, types, trailer): - trailer_op, node = trailer.children[:2] - if node == ')': # `arglist` is optional. - node = () - - new_types = set() - if trailer_op == '[': - new_types |= iterable.py__getitem__(self, types, trailer) - else: - for typ in types: - debug.dbg('eval_trailer: %s in scope %s', trailer, typ) - if trailer_op == '.': - new_types |= self.find_types(typ, node) - elif trailer_op == '(': - new_types |= self.execute(typ, node, trailer) - return new_types - - def execute_evaluated(self, obj, *args): - """ - Execute a function with already executed arguments. - """ - args = [iterable.AlreadyEvaluated([arg]) for arg in args] - return self.execute(obj, args) - - @debug.increase_indent - def execute(self, obj, arguments=(), trailer=None): - if not isinstance(arguments, param.Arguments): - arguments = param.Arguments(self, arguments, trailer) - - if self.is_analysis: - arguments.eval_all() - - if obj.isinstance(er.Function): - obj = obj.get_decorated_func() - - debug.dbg('execute: %s %s', obj, arguments) - try: - # Some stdlib functions like super(), namedtuple(), etc. have been - # hard-coded in Jedi to support them. - return stdlib.execute(self, obj, arguments) - except stdlib.NotInStdLib: - pass - - try: - func = obj.py__call__ - except AttributeError: - debug.warning("no execution possible %s", obj) - return set() - else: - types = func(arguments) - debug.dbg('execute result: %s in %s', types, obj) - return types - - def goto_definitions(self, name): - def_ = name.get_definition() - is_simple_name = name.parent.type not in ('power', 'trailer') - if is_simple_name: - if name.parent.type == 'classdef' and name.parent.name == name: - return [self.wrap(name.parent)] - if name.parent.type in ('file_input', 'funcdef'): - return [self.wrap(name.parent)] - if def_.type == 'expr_stmt' and name in def_.get_defined_names(): - return self.eval_statement(def_, name) - elif def_.type == 'for_stmt': - container_types = self.eval_element(def_.children[3]) - for_types = iterable.py__iter__types(self, container_types, def_.children[3]) - return finder.check_tuple_assignments(self, for_types, name) - elif def_.type in ('import_from', 'import_name'): - return imports.ImportWrapper(self, name).follow() - - call = helpers.call_of_leaf(name) - return self.eval_element(call) - - def goto(self, name): - def resolve_implicit_imports(names): - for name in names: - if isinstance(name.parent, helpers.FakeImport): - # Those are implicit imports. - s = imports.ImportWrapper(self, name) - for n in s.follow(is_goto=True): - yield n - else: - yield name - - stmt = name.get_definition() - par = name.parent - if par.type == 'argument' and par.children[1] == '=' and par.children[0] == name: - # Named param goto. - trailer = par.parent - if trailer.type == 'arglist': - trailer = trailer.parent - if trailer.type != 'classdef': - if trailer.type == 'decorator': - types = self.eval_element(trailer.children[1]) - else: - i = trailer.parent.children.index(trailer) - to_evaluate = trailer.parent.children[:i] - types = self.eval_element(to_evaluate[0]) - for trailer in to_evaluate[1:]: - types = self.eval_trailer(types, trailer) - param_names = [] - for typ in types: - try: - params = typ.params - except AttributeError: - pass - else: - param_names += [param.name for param in params - if param.name.value == name.value] - return param_names - elif isinstance(par, tree.ExprStmt) and name in par.get_defined_names(): - # Only take the parent, because if it's more complicated than just - # a name it's something you can "goto" again. - return [name] - elif isinstance(par, (tree.Param, tree.Function, tree.Class)) and par.name is name: - return [name] - elif isinstance(stmt, tree.Import): - modules = imports.ImportWrapper(self, name).follow(is_goto=True) - return list(resolve_implicit_imports(modules)) - elif par.type == 'dotted_name': # Is a decorator. - index = par.children.index(name) - if index > 0: - new_dotted = helpers.deep_ast_copy(par) - new_dotted.children[index - 1:] = [] - types = self.eval_element(new_dotted) - return resolve_implicit_imports(iterable.unite( - self.find_types(typ, name, is_goto=True) for typ in types - )) - - scope = name.get_parent_scope() - if tree.is_node(par, 'trailer') and par.children[0] == '.': - call = helpers.call_of_leaf(name, cut_own_trailer=True) - types = self.eval_element(call) - return resolve_implicit_imports(iterable.unite( - self.find_types(typ, name, is_goto=True) for typ in types - )) - else: - if stmt.type != 'expr_stmt': - # We only need to adjust the start_pos for statements, because - # there the name cannot be used. - stmt = name - return self.find_types(scope, name, stmt.start_pos, - search_global=True, is_goto=True) diff --git a/pythonFiles/preview/jedi/evaluate/analysis.py b/pythonFiles/preview/jedi/evaluate/analysis.py deleted file mode 100644 index 407bc7da464c..000000000000 --- a/pythonFiles/preview/jedi/evaluate/analysis.py +++ /dev/null @@ -1,216 +0,0 @@ -""" -Module for statical analysis. -""" -from jedi import debug -from jedi.parser import tree -from jedi.evaluate.compiled import CompiledObject - -from jedi.common import unite - - -CODES = { - 'attribute-error': (1, AttributeError, 'Potential AttributeError.'), - 'name-error': (2, NameError, 'Potential NameError.'), - 'import-error': (3, ImportError, 'Potential ImportError.'), - 'type-error-too-many-arguments': (4, TypeError, None), - 'type-error-too-few-arguments': (5, TypeError, None), - 'type-error-keyword-argument': (6, TypeError, None), - 'type-error-multiple-values': (7, TypeError, None), - 'type-error-star-star': (8, TypeError, None), - 'type-error-star': (9, TypeError, None), - 'type-error-operation': (10, TypeError, None), - 'type-error-not-iterable': (11, TypeError, None), - 'type-error-isinstance': (12, TypeError, None), - 'type-error-not-subscriptable': (13, TypeError, None), - 'value-error-too-many-values': (14, ValueError, None), - 'value-error-too-few-values': (15, ValueError, None), -} - - -class Error(object): - def __init__(self, name, module_path, start_pos, message=None): - self.path = module_path - self._start_pos = start_pos - self.name = name - if message is None: - message = CODES[self.name][2] - self.message = message - - @property - def line(self): - return self._start_pos[0] - - @property - def column(self): - return self._start_pos[1] - - @property - def code(self): - # The class name start - first = self.__class__.__name__[0] - return first + str(CODES[self.name][0]) - - def __unicode__(self): - return '%s:%s:%s: %s %s' % (self.path, self.line, self.column, - self.code, self.message) - - def __str__(self): - return self.__unicode__() - - def __eq__(self, other): - return (self.path == other.path and self.name == other.name - and self._start_pos == other._start_pos) - - def __ne__(self, other): - return not self.__eq__(other) - - def __hash__(self): - return hash((self.path, self._start_pos, self.name)) - - def __repr__(self): - return '<%s %s: %s@%s,%s>' % (self.__class__.__name__, - self.name, self.path, - self._start_pos[0], self._start_pos[1]) - - -class Warning(Error): - pass - - -def add(evaluator, name, jedi_obj, message=None, typ=Error, payload=None): - from jedi.evaluate.iterable import MergedNodes - while isinstance(jedi_obj, MergedNodes): - if len(jedi_obj) != 1: - # TODO is this kosher? - return - jedi_obj = list(jedi_obj)[0] - - exception = CODES[name][1] - if _check_for_exception_catch(evaluator, jedi_obj, exception, payload): - return - - module_path = jedi_obj.get_parent_until().path - instance = typ(name, module_path, jedi_obj.start_pos, message) - debug.warning(str(instance), format=False) - evaluator.analysis.append(instance) - - -def _check_for_setattr(instance): - """ - Check if there's any setattr method inside an instance. If so, return True. - """ - module = instance.get_parent_until() - try: - stmts = module.used_names['setattr'] - except KeyError: - return False - - return any(instance.start_pos < stmt.start_pos < instance.end_pos - for stmt in stmts) - - -def add_attribute_error(evaluator, scope, name): - message = ('AttributeError: %s has no attribute %s.' % (scope, name)) - from jedi.evaluate.representation import Instance - # Check for __getattr__/__getattribute__ existance and issue a warning - # instead of an error, if that happens. - if isinstance(scope, Instance): - typ = Warning - try: - scope.get_subscope_by_name('__getattr__') - except KeyError: - try: - scope.get_subscope_by_name('__getattribute__') - except KeyError: - if not _check_for_setattr(scope): - typ = Error - else: - typ = Error - - payload = scope, name - add(evaluator, 'attribute-error', name, message, typ, payload) - - -def _check_for_exception_catch(evaluator, jedi_obj, exception, payload=None): - """ - Checks if a jedi object (e.g. `Statement`) sits inside a try/catch and - doesn't count as an error (if equal to `exception`). - Also checks `hasattr` for AttributeErrors and uses the `payload` to compare - it. - Returns True if the exception was catched. - """ - def check_match(cls, exception): - try: - return isinstance(cls, CompiledObject) and issubclass(exception, cls.obj) - except TypeError: - return False - - def check_try_for_except(obj, exception): - # Only nodes in try - iterator = iter(obj.children) - for branch_type in iterator: - colon = next(iterator) - suite = next(iterator) - if branch_type == 'try' \ - and not (branch_type.start_pos < jedi_obj.start_pos <= suite.end_pos): - return False - - for node in obj.except_clauses(): - if node is None: - return True # An exception block that catches everything. - else: - except_classes = evaluator.eval_element(node) - for cls in except_classes: - from jedi.evaluate import iterable - if isinstance(cls, iterable.Array) and cls.type == 'tuple': - # multiple exceptions - for typ in unite(cls.py__iter__()): - if check_match(typ, exception): - return True - else: - if check_match(cls, exception): - return True - - def check_hasattr(node, suite): - try: - assert suite.start_pos <= jedi_obj.start_pos < suite.end_pos - assert node.type in ('power', 'atom_expr') - base = node.children[0] - assert base.type == 'name' and base.value == 'hasattr' - trailer = node.children[1] - assert trailer.type == 'trailer' - arglist = trailer.children[1] - assert arglist.type == 'arglist' - from jedi.evaluate.param import Arguments - args = list(Arguments(evaluator, arglist).unpack()) - # Arguments should be very simple - assert len(args) == 2 - - # Check name - key, values = args[1] - assert len(values) == 1 - names = list(evaluator.eval_element(values[0])) - assert len(names) == 1 and isinstance(names[0], CompiledObject) - assert names[0].obj == str(payload[1]) - - # Check objects - key, values = args[0] - assert len(values) == 1 - objects = evaluator.eval_element(values[0]) - return payload[0] in objects - except AssertionError: - return False - - obj = jedi_obj - while obj is not None and not obj.isinstance(tree.Function, tree.Class): - if obj.isinstance(tree.Flow): - # try/except catch check - if obj.isinstance(tree.TryStmt) and check_try_for_except(obj, exception): - return True - # hasattr check - if exception == AttributeError and obj.isinstance(tree.IfStmt, tree.WhileStmt): - if check_hasattr(obj.children[1], obj.children[3]): - return True - obj = obj.parent - - return False diff --git a/pythonFiles/preview/jedi/evaluate/cache.py b/pythonFiles/preview/jedi/evaluate/cache.py deleted file mode 100644 index b44f812accd2..000000000000 --- a/pythonFiles/preview/jedi/evaluate/cache.py +++ /dev/null @@ -1,58 +0,0 @@ -""" -- the popular ``memoize_default`` works like a typical memoize and returns the - default otherwise. -- ``CachedMetaClass`` uses ``memoize_default`` to do the same with classes. -""" - -import inspect - -NO_DEFAULT = object() - - -def memoize_default(default=NO_DEFAULT, evaluator_is_first_arg=False, second_arg_is_evaluator=False): - """ This is a typical memoization decorator, BUT there is one difference: - To prevent recursion it sets defaults. - - Preventing recursion is in this case the much bigger use than speed. I - don't think, that there is a big speed difference, but there are many cases - where recursion could happen (think about a = b; b = a). - """ - def func(function): - def wrapper(obj, *args, **kwargs): - if evaluator_is_first_arg: - cache = obj.memoize_cache - elif second_arg_is_evaluator: # needed for meta classes - cache = args[0].memoize_cache - else: - cache = obj._evaluator.memoize_cache - - try: - memo = cache[function] - except KeyError: - memo = {} - cache[function] = memo - - key = (obj, args, frozenset(kwargs.items())) - if key in memo: - return memo[key] - else: - if default is not NO_DEFAULT: - memo[key] = default - rv = function(obj, *args, **kwargs) - if inspect.isgenerator(rv): - rv = list(rv) - memo[key] = rv - return rv - return wrapper - return func - - -class CachedMetaClass(type): - """ - This is basically almost the same than the decorator above, it just caches - class initializations. Either you do it this way or with decorators, but - with decorators you lose class access (isinstance, etc). - """ - @memoize_default(None, second_arg_is_evaluator=True) - def __call__(self, *args, **kwargs): - return super(CachedMetaClass, self).__call__(*args, **kwargs) diff --git a/pythonFiles/preview/jedi/evaluate/compiled/__init__.py b/pythonFiles/preview/jedi/evaluate/compiled/__init__.py deleted file mode 100644 index f004a142a7e4..000000000000 --- a/pythonFiles/preview/jedi/evaluate/compiled/__init__.py +++ /dev/null @@ -1,544 +0,0 @@ -""" -Imitate the parser representation. -""" -import inspect -import re -import sys -import os -from functools import partial - -from jedi._compatibility import builtins as _builtins, unicode -from jedi import debug -from jedi.cache import underscore_memoization, memoize_method -from jedi.parser.tree import Param, Base, Operator, zero_position_modifier -from jedi.evaluate.helpers import FakeName -from . import fake - - -_sep = os.path.sep -if os.path.altsep is not None: - _sep += os.path.altsep -_path_re = re.compile('(?:\.[^{0}]+|[{0}]__init__\.py)$'.format(re.escape(_sep))) -del _sep - - -class CheckAttribute(object): - """Raises an AttributeError if the attribute X isn't available.""" - def __init__(self, func): - self.func = func - # Remove the py in front of e.g. py__call__. - self.check_name = func.__name__[2:] - - def __get__(self, instance, owner): - # This might raise an AttributeError. That's wanted. - getattr(instance.obj, self.check_name) - return partial(self.func, instance) - - -class CompiledObject(Base): - # comply with the parser - start_pos = 0, 0 - path = None # modules have this attribute - set it to None. - used_names = {} # To be consistent with modules. - - def __init__(self, evaluator, obj, parent=None): - self._evaluator = evaluator - self.obj = obj - self.parent = parent - - @CheckAttribute - def py__call__(self, params): - if inspect.isclass(self.obj): - from jedi.evaluate.representation import Instance - return set([Instance(self._evaluator, self, params)]) - else: - return set(self._execute_function(params)) - - @CheckAttribute - def py__class__(self): - return create(self._evaluator, self.obj.__class__) - - @CheckAttribute - def py__mro__(self): - return tuple(create(self._evaluator, cls) for cls in self.obj.__mro__) - - @CheckAttribute - def py__bases__(self): - return tuple(create(self._evaluator, cls) for cls in self.obj.__bases__) - - def py__bool__(self): - return bool(self.obj) - - def py__file__(self): - return self.obj.__file__ - - def is_class(self): - return inspect.isclass(self.obj) - - @property - def doc(self): - return inspect.getdoc(self.obj) or '' - - @property - def params(self): - params_str, ret = self._parse_function_doc() - tokens = params_str.split(',') - if inspect.ismethoddescriptor(self.obj): - tokens.insert(0, 'self') - params = [] - for p in tokens: - parts = [FakeName(part) for part in p.strip().split('=')] - if len(parts) > 1: - parts.insert(1, Operator(zero_position_modifier, '=', (0, 0))) - params.append(Param(parts, self)) - return params - - def __repr__(self): - return '<%s: %s>' % (type(self).__name__, repr(self.obj)) - - @underscore_memoization - def _parse_function_doc(self): - if self.doc is None: - return '', '' - - return _parse_function_doc(self.doc) - - def api_type(self): - obj = self.obj - if inspect.isclass(obj): - return 'class' - elif inspect.ismodule(obj): - return 'module' - elif inspect.isbuiltin(obj) or inspect.ismethod(obj) \ - or inspect.ismethoddescriptor(obj) or inspect.isfunction(obj): - return 'function' - # Everything else... - return 'instance' - - @property - def type(self): - """Imitate the tree.Node.type values.""" - cls = self._get_class() - if inspect.isclass(cls): - return 'classdef' - elif inspect.ismodule(cls): - return 'file_input' - elif inspect.isbuiltin(cls) or inspect.ismethod(cls) or \ - inspect.ismethoddescriptor(cls): - return 'funcdef' - - @underscore_memoization - def _cls(self): - """ - We used to limit the lookups for instantiated objects like list(), but - this is not the case anymore. Python itself - """ - # Ensures that a CompiledObject is returned that is not an instance (like list) - return self - - def _get_class(self): - if not fake.is_class_instance(self.obj) or \ - inspect.ismethoddescriptor(self.obj): # slots - return self.obj - - try: - return self.obj.__class__ - except AttributeError: - # happens with numpy.core.umath._UFUNC_API (you get it - # automatically by doing `import numpy`. - return type - - @property - def names_dict(self): - # For compatibility with `representation.Class`. - return self.names_dicts(False)[0] - - def names_dicts(self, search_global, is_instance=False): - return self._names_dict_ensure_one_dict(is_instance) - - @memoize_method - def _names_dict_ensure_one_dict(self, is_instance): - """ - search_global shouldn't change the fact that there's one dict, this way - there's only one `object`. - """ - return [LazyNamesDict(self._evaluator, self, is_instance)] - - def get_subscope_by_name(self, name): - if name in dir(self.obj): - return CompiledName(self._evaluator, self, name).parent - else: - raise KeyError("CompiledObject doesn't have an attribute '%s'." % name) - - @CheckAttribute - def py__getitem__(self, index): - if type(self.obj) not in (str, list, tuple, unicode, bytes, bytearray, dict): - # Get rid of side effects, we won't call custom `__getitem__`s. - return set() - - return set([create(self._evaluator, self.obj[index])]) - - @CheckAttribute - def py__iter__(self): - if type(self.obj) not in (str, list, tuple, unicode, bytes, bytearray, dict): - # Get rid of side effects, we won't call custom `__getitem__`s. - return - - for part in self.obj: - yield set([create(self._evaluator, part)]) - - @property - def name(self): - try: - name = self._get_class().__name__ - except AttributeError: - name = repr(self.obj) - return FakeName(name, self) - - def _execute_function(self, params): - if self.type != 'funcdef': - return - - for name in self._parse_function_doc()[1].split(): - try: - bltn_obj = getattr(_builtins, name) - except AttributeError: - continue - else: - if bltn_obj is None: - # We want to evaluate everything except None. - # TODO do we? - continue - bltn_obj = create(self._evaluator, bltn_obj) - for result in self._evaluator.execute(bltn_obj, params): - yield result - - @property - @underscore_memoization - def subscopes(self): - """ - Returns only the faked scopes - the other ones are not important for - internal analysis. - """ - module = self.get_parent_until() - faked_subscopes = [] - for name in dir(self.obj): - try: - faked_subscopes.append( - fake.get_faked(module.obj, self.obj, parent=self, name=name) - ) - except fake.FakeDoesNotExist: - pass - return faked_subscopes - - def is_scope(self): - return True - - def get_self_attributes(self): - return [] # Instance compatibility - - def get_imports(self): - return [] # Builtins don't have imports - - -class CompiledName(FakeName): - def __init__(self, evaluator, compiled_obj, name): - super(CompiledName, self).__init__(name) - self._evaluator = evaluator - self._compiled_obj = compiled_obj - self.name = name - - def __repr__(self): - try: - name = self._compiled_obj.name # __name__ is not defined all the time - except AttributeError: - name = None - return '<%s: (%s).%s>' % (type(self).__name__, name, self.name) - - def is_definition(self): - return True - - @property - @underscore_memoization - def parent(self): - module = self._compiled_obj.get_parent_until() - return _create_from_name(self._evaluator, module, self._compiled_obj, self.name) - - @parent.setter - def parent(self, value): - pass # Just ignore this, FakeName tries to overwrite the parent attribute. - - -class LazyNamesDict(object): - """ - A names_dict instance for compiled objects, resembles the parser.tree. - """ - name_class = CompiledName - - def __init__(self, evaluator, compiled_obj, is_instance=False): - self._evaluator = evaluator - self._compiled_obj = compiled_obj - self._is_instance = is_instance - - def __iter__(self): - return (v[0].value for v in self.values()) - - @memoize_method - def __getitem__(self, name): - try: - getattr(self._compiled_obj.obj, name) - except AttributeError: - raise KeyError('%s in %s not found.' % (name, self._compiled_obj)) - except Exception: - # This is a bit ugly. We're basically returning this to make - # lookups possible without having the actual attribute. However - # this makes proper completion possible. - return [FakeName(name, create(self._evaluator, None), is_definition=True)] - return [self.name_class(self._evaluator, self._compiled_obj, name)] - - def values(self): - obj = self._compiled_obj.obj - - values = [] - for name in dir(obj): - try: - values.append(self[name]) - except KeyError: - # The dir function can be wrong. - pass - - is_instance = self._is_instance or fake.is_class_instance(obj) - # ``dir`` doesn't include the type names. - if not inspect.ismodule(obj) and obj != type and not is_instance: - values += create(self._evaluator, type).names_dict.values() - return values - - -def dotted_from_fs_path(fs_path, sys_path): - """ - Changes `/usr/lib/python3.4/email/utils.py` to `email.utils`. I.e. - compares the path with sys.path and then returns the dotted_path. If the - path is not in the sys.path, just returns None. - """ - if os.path.basename(fs_path).startswith('__init__.'): - # We are calculating the path. __init__ files are not interesting. - fs_path = os.path.dirname(fs_path) - - # prefer - # - UNIX - # /path/to/pythonX.Y/lib-dynload - # /path/to/pythonX.Y/site-packages - # - Windows - # C:\path\to\DLLs - # C:\path\to\Lib\site-packages - # over - # - UNIX - # /path/to/pythonX.Y - # - Windows - # C:\path\to\Lib - path = '' - for s in sys_path: - if (fs_path.startswith(s) and len(path) < len(s)): - path = s - - # - Window - # X:\path\to\lib-dynload/datetime.pyd => datetime - module_path = fs_path[len(path):].lstrip(os.path.sep).lstrip('/') - # - Window - # Replace like X:\path\to\something/foo/bar.py - return _path_re.sub('', module_path).replace(os.path.sep, '.').replace('/', '.') - - -def load_module(evaluator, path=None, name=None): - sys_path = evaluator.sys_path - if path is not None: - dotted_path = dotted_from_fs_path(path, sys_path=sys_path) - else: - dotted_path = name - - if dotted_path is None: - p, _, dotted_path = path.partition(os.path.sep) - sys_path.insert(0, p) - - temp, sys.path = sys.path, sys_path - try: - __import__(dotted_path) - except RuntimeError: - if 'PySide' in dotted_path or 'PyQt' in dotted_path: - # RuntimeError: the PyQt4.QtCore and PyQt5.QtCore modules both wrap - # the QObject class. - # See https://github.com/davidhalter/jedi/pull/483 - return None - raise - except ImportError: - # If a module is "corrupt" or not really a Python module or whatever. - debug.warning('Module %s not importable.', path) - return None - finally: - sys.path = temp - - # Just access the cache after import, because of #59 as well as the very - # complicated import structure of Python. - module = sys.modules[dotted_path] - - return create(evaluator, module) - - -docstr_defaults = { - 'floating point number': 'float', - 'character': 'str', - 'integer': 'int', - 'dictionary': 'dict', - 'string': 'str', -} - - -def _parse_function_doc(doc): - """ - Takes a function and returns the params and return value as a tuple. - This is nothing more than a docstring parser. - - TODO docstrings like utime(path, (atime, mtime)) and a(b [, b]) -> None - TODO docstrings like 'tuple of integers' - """ - # parse round parentheses: def func(a, (b,c)) - try: - count = 0 - start = doc.index('(') - for i, s in enumerate(doc[start:]): - if s == '(': - count += 1 - elif s == ')': - count -= 1 - if count == 0: - end = start + i - break - param_str = doc[start + 1:end] - except (ValueError, UnboundLocalError): - # ValueError for doc.index - # UnboundLocalError for undefined end in last line - debug.dbg('no brackets found - no param') - end = 0 - param_str = '' - else: - # remove square brackets, that show an optional param ( = None) - def change_options(m): - args = m.group(1).split(',') - for i, a in enumerate(args): - if a and '=' not in a: - args[i] += '=None' - return ','.join(args) - - while True: - param_str, changes = re.subn(r' ?\[([^\[\]]+)\]', - change_options, param_str) - if changes == 0: - break - param_str = param_str.replace('-', '_') # see: isinstance.__doc__ - - # parse return value - r = re.search('-[>-]* ', doc[end:end + 7]) - if r is None: - ret = '' - else: - index = end + r.end() - # get result type, which can contain newlines - pattern = re.compile(r'(,\n|[^\n-])+') - ret_str = pattern.match(doc, index).group(0).strip() - # New object -> object() - ret_str = re.sub(r'[nN]ew (.*)', r'\1()', ret_str) - - ret = docstr_defaults.get(ret_str, ret_str) - - return param_str, ret - - -def _create_from_name(evaluator, module, parent, name): - try: - return fake.get_faked(module.obj, parent.obj, parent=parent, name=name) - except fake.FakeDoesNotExist: - pass - - try: - obj = getattr(parent.obj, name) - except AttributeError: - # Happens e.g. in properties of - # PyQt4.QtGui.QStyleOptionComboBox.currentText - # -> just set it to None - obj = None - return create(evaluator, obj, parent) - - -def builtin_from_name(evaluator, string): - bltn_obj = getattr(_builtins, string) - return create(evaluator, bltn_obj) - - -def _a_generator(foo): - """Used to have an object to return for generators.""" - yield 42 - yield foo - - -_SPECIAL_OBJECTS = { - 'FUNCTION_CLASS': type(load_module), - 'METHOD_CLASS': type(CompiledObject.is_class), - 'MODULE_CLASS': type(os), - 'GENERATOR_OBJECT': _a_generator(1.0), - 'BUILTINS': _builtins, -} - - -def get_special_object(evaluator, identifier): - obj = _SPECIAL_OBJECTS[identifier] - return create(evaluator, obj, parent=create(evaluator, _builtins)) - - -def compiled_objects_cache(attribute_name): - def decorator(func): - """ - This decorator caches just the ids, oopposed to caching the object itself. - Caching the id has the advantage that an object doesn't need to be - hashable. - """ - def wrapper(evaluator, obj, parent=None, module=None): - cache = getattr(evaluator, attribute_name) - # Do a very cheap form of caching here. - key = id(obj), id(parent) - try: - return cache[key][0] - except KeyError: - # TODO this whole decorator looks way too ugly and this if - # doesn't make it better. Find a more generic solution. - if parent or module: - result = func(evaluator, obj, parent, module) - else: - result = func(evaluator, obj) - # Need to cache all of them, otherwise the id could be overwritten. - cache[key] = result, obj, parent, module - return result - return wrapper - - return decorator - - -@compiled_objects_cache('compiled_cache') -def create(evaluator, obj, parent=None, module=None): - """ - A very weird interface class to this module. The more options provided the - more acurate loading compiled objects is. - """ - if inspect.ismodule(obj): - if parent is not None: - # Modules don't have parents, be careful with caching: recurse. - return create(evaluator, obj) - else: - if parent is None and obj != _builtins: - return create(evaluator, obj, create(evaluator, _builtins)) - - try: - return fake.get_faked(module and module.obj, obj, parent=parent) - except fake.FakeDoesNotExist: - pass - - return CompiledObject(evaluator, obj, parent) diff --git a/pythonFiles/preview/jedi/evaluate/compiled/fake.py b/pythonFiles/preview/jedi/evaluate/compiled/fake.py deleted file mode 100644 index 0338b962335c..000000000000 --- a/pythonFiles/preview/jedi/evaluate/compiled/fake.py +++ /dev/null @@ -1,203 +0,0 @@ -""" -Loads functions that are mixed in to the standard library. E.g. builtins are -written in C (binaries), but my autocompletion only understands Python code. By -mixing in Python code, the autocompletion should work much better for builtins. -""" - -import os -import inspect -import types - -from jedi._compatibility import is_py3, builtins, unicode, is_py34 -from jedi.parser import ParserWithRecovery, load_grammar -from jedi.parser import tree as pt -from jedi.evaluate.helpers import FakeName - -modules = {} - - -MethodDescriptorType = type(str.replace) -# These are not considered classes and access is granted even though they have -# a __class__ attribute. -NOT_CLASS_TYPES = ( - types.BuiltinFunctionType, - types.CodeType, - types.FrameType, - types.FunctionType, - types.GeneratorType, - types.GetSetDescriptorType, - types.LambdaType, - types.MemberDescriptorType, - types.MethodType, - types.ModuleType, - types.TracebackType, - MethodDescriptorType -) - -if is_py3: - NOT_CLASS_TYPES += ( - types.MappingProxyType, - types.SimpleNamespace - ) - if is_py34: - NOT_CLASS_TYPES += (types.DynamicClassAttribute,) - - -class FakeDoesNotExist(Exception): - pass - - -def _load_faked_module(module): - module_name = module.__name__ - if module_name == '__builtin__' and not is_py3: - module_name = 'builtins' - - try: - return modules[module_name] - except KeyError: - path = os.path.dirname(os.path.abspath(__file__)) - try: - with open(os.path.join(path, 'fake', module_name) + '.pym') as f: - source = f.read() - except IOError: - modules[module_name] = None - return - grammar = load_grammar(version='3.4') - module = ParserWithRecovery(grammar, unicode(source), module_name).module - modules[module_name] = module - - if module_name == 'builtins' and not is_py3: - # There are two implementations of `open` for either python 2/3. - # -> Rename the python2 version (`look at fake/builtins.pym`). - open_func = search_scope(module, 'open') - open_func.children[1] = FakeName('open_python3') - open_func = search_scope(module, 'open_python2') - open_func.children[1] = FakeName('open') - return module - - -def search_scope(scope, obj_name): - for s in scope.subscopes: - if str(s.name) == obj_name: - return s - - -def get_module(obj): - if inspect.ismodule(obj): - return obj - try: - obj = obj.__objclass__ - except AttributeError: - pass - - try: - imp_plz = obj.__module__ - except AttributeError: - # Unfortunately in some cases like `int` there's no __module__ - return builtins - else: - if imp_plz is None: - # Happens for example in `(_ for _ in []).send.__module__`. - return builtins - else: - try: - return __import__(imp_plz) - except ImportError: - # __module__ can be something arbitrary that doesn't exist. - return builtins - - -def _faked(module, obj, name): - # Crazy underscore actions to try to escape all the internal madness. - if module is None: - module = get_module(obj) - - faked_mod = _load_faked_module(module) - if faked_mod is None: - return None - - # Having the module as a `parser.representation.module`, we need to scan - # for methods. - if name is None: - if inspect.isbuiltin(obj): - return search_scope(faked_mod, obj.__name__) - elif not inspect.isclass(obj): - # object is a method or descriptor - try: - objclass = obj.__objclass__ - except AttributeError: - return None - else: - cls = search_scope(faked_mod, objclass.__name__) - if cls is None: - return None - return search_scope(cls, obj.__name__) - else: - if obj == module: - return search_scope(faked_mod, name) - else: - try: - cls_name = obj.__name__ - except AttributeError: - return None - cls = search_scope(faked_mod, cls_name) - if cls is None: - return None - return search_scope(cls, name) - - -def memoize_faked(obj): - """ - A typical memoize function that ignores issues with non hashable results. - """ - cache = obj.cache = {} - - def memoizer(*args, **kwargs): - key = (obj, args, frozenset(kwargs.items())) - try: - result = cache[key] - except TypeError: - return obj(*args, **kwargs) - except KeyError: - result = obj(*args, **kwargs) - if result is not None: - cache[key] = obj(*args, **kwargs) - return result - else: - return result - return memoizer - - -@memoize_faked -def _get_faked(module, obj, name=None): - obj = type(obj) if is_class_instance(obj) else obj - result = _faked(module, obj, name) - if result is None or isinstance(result, pt.Class): - # We're not interested in classes. What we want is functions. - raise FakeDoesNotExist - else: - # Set the docstr which was previously not set (faked modules don't - # contain it). - doc = '"""%s"""' % obj.__doc__ # TODO need escapes. - suite = result.children[-1] - string = pt.String(pt.zero_position_modifier, doc, (0, 0), '') - new_line = pt.Newline('\n', (0, 0), '') - docstr_node = pt.Node('simple_stmt', [string, new_line]) - suite.children.insert(2, docstr_node) - return result - - -def get_faked(module, obj, name=None, parent=None): - faked = _get_faked(module, obj, name) - faked.parent = parent - return faked - - -def is_class_instance(obj): - """Like inspect.* methods.""" - try: - cls = obj.__class__ - except AttributeError: - return False - else: - return cls != type and not issubclass(cls, NOT_CLASS_TYPES) diff --git a/pythonFiles/preview/jedi/evaluate/compiled/fake/_weakref.pym b/pythonFiles/preview/jedi/evaluate/compiled/fake/_weakref.pym deleted file mode 100644 index 8d21a2c4a7c6..000000000000 --- a/pythonFiles/preview/jedi/evaluate/compiled/fake/_weakref.pym +++ /dev/null @@ -1,8 +0,0 @@ -def proxy(object, callback=None): - return object - -class weakref(): - def __init__(self, object, callback=None): - self.__object = object - def __call__(self): - return self.__object diff --git a/pythonFiles/preview/jedi/evaluate/compiled/fake/builtins.pym b/pythonFiles/preview/jedi/evaluate/compiled/fake/builtins.pym deleted file mode 100644 index 1d5314bde97a..000000000000 --- a/pythonFiles/preview/jedi/evaluate/compiled/fake/builtins.pym +++ /dev/null @@ -1,262 +0,0 @@ -""" -Pure Python implementation of some builtins. -This code is not going to be executed anywhere. -These implementations are not always correct, but should work as good as -possible for the auto completion. -""" - - -def next(iterator, default=None): - if random.choice([0, 1]): - if hasattr("next"): - return iterator.next() - else: - return iterator.__next__() - else: - if default is not None: - return default - - -def iter(collection, sentinel=None): - if sentinel: - yield collection() - else: - for c in collection: - yield c - - -def range(start, stop=None, step=1): - return [0] - - -class file(): - def __iter__(self): - yield '' - def next(self): - return '' - - -class xrange(): - # Attention: this function doesn't exist in Py3k (there it is range). - def __iter__(self): - yield 1 - - def count(self): - return 1 - - def index(self): - return 1 - - -def open(file, mode='r', buffering=-1, encoding=None, errors=None, newline=None, closefd=True): - import io - return io.TextIOWrapper(file, mode, buffering, encoding, errors, newline, closefd) - - -def open_python2(name, mode=None, buffering=None): - return file(name, mode, buffering) - - -#-------------------------------------------------------- -# descriptors -#-------------------------------------------------------- -class property(): - def __init__(self, fget, fset=None, fdel=None, doc=None): - self.fget = fget - self.fset = fset - self.fdel = fdel - self.__doc__ = doc - - def __get__(self, obj, cls): - return self.fget(obj) - - def __set__(self, obj, value): - self.fset(obj, value) - - def __delete__(self, obj): - self.fdel(obj) - - def setter(self, func): - self.fset = func - return self - - def getter(self, func): - self.fget = func - return self - - def deleter(self, func): - self.fdel = func - return self - - -class staticmethod(): - def __init__(self, func): - self.__func = func - - def __get__(self, obj, cls): - return self.__func - - -class classmethod(): - def __init__(self, func): - self.__func = func - - def __get__(self, obj, cls): - def _method(*args, **kwargs): - return self.__func(cls, *args, **kwargs) - return _method - - -#-------------------------------------------------------- -# array stuff -#-------------------------------------------------------- -class list(): - def __init__(self, iterable=[]): - self.__iterable = [] - for i in iterable: - self.__iterable += [i] - - def __iter__(self): - for i in self.__iterable: - yield i - - def __getitem__(self, y): - return self.__iterable[y] - - def pop(self): - return self.__iterable[int()] - - -class tuple(): - def __init__(self, iterable=[]): - self.__iterable = [] - for i in iterable: - self.__iterable += [i] - - def __iter__(self): - for i in self.__iterable: - yield i - - def __getitem__(self, y): - return self.__iterable[y] - - def index(self): - return 1 - - def count(self): - return 1 - - -class set(): - def __init__(self, iterable=[]): - self.__iterable = iterable - - def __iter__(self): - for i in self.__iterable: - yield i - - def pop(self): - return list(self.__iterable)[-1] - - def copy(self): - return self - - def difference(self, other): - return self - other - - def intersection(self, other): - return self & other - - def symmetric_difference(self, other): - return self ^ other - - def union(self, other): - return self | other - - -class frozenset(): - def __init__(self, iterable=[]): - self.__iterable = iterable - - def __iter__(self): - for i in self.__iterable: - yield i - - def copy(self): - return self - - -class dict(): - def __init__(self, **elements): - self.__elements = elements - - def clear(self): - # has a strange docstr - pass - - def get(self, k, d=None): - # TODO implement - try: - #return self.__elements[k] - pass - except KeyError: - return d - - def values(self): - return self.__elements.values() - - def setdefault(self, k, d): - # TODO maybe also return the content - return d - - -class enumerate(): - def __init__(self, sequence, start=0): - self.__sequence = sequence - - def __iter__(self): - for i in self.__sequence: - yield 1, i - - def __next__(self): - return next(self.__iter__()) - - def next(self): - return next(self.__iter__()) - - -class reversed(): - def __init__(self, sequence): - self.__sequence = sequence - - def __iter__(self): - for i in self.__sequence: - yield i - - def __next__(self): - return next(self.__iter__()) - - def next(self): - return next(self.__iter__()) - - -def sorted(iterable, cmp=None, key=None, reverse=False): - return iterable - - -#-------------------------------------------------------- -# basic types -#-------------------------------------------------------- -class int(): - def __init__(self, x, base=None): - pass - - -class str(): - def __init__(self, obj): - pass - - -class type(): - def mro(): - return [object] diff --git a/pythonFiles/preview/jedi/evaluate/compiled/fake/io.pym b/pythonFiles/preview/jedi/evaluate/compiled/fake/io.pym deleted file mode 100644 index 87b02eed4321..000000000000 --- a/pythonFiles/preview/jedi/evaluate/compiled/fake/io.pym +++ /dev/null @@ -1,6 +0,0 @@ -class TextIOWrapper(): - def __next__(self): - return str() - - def __iter__(self): - yield str() diff --git a/pythonFiles/preview/jedi/evaluate/compiled/mixed.py b/pythonFiles/preview/jedi/evaluate/compiled/mixed.py deleted file mode 100644 index 69119f181f6d..000000000000 --- a/pythonFiles/preview/jedi/evaluate/compiled/mixed.py +++ /dev/null @@ -1,158 +0,0 @@ -""" -Used only for REPL Completion. -""" - -import inspect -import os - -from jedi import common -from jedi.parser.fast import FastParser -from jedi.evaluate import compiled -from jedi.cache import underscore_memoization - - -class MixedObject(object): - """ - A ``MixedObject`` is used in two ways: - - 1. It uses the default logic of ``parser.tree`` objects, - 2. except for getattr calls. The names dicts are generated in a fashion - like ``CompiledObject``. - - This combined logic makes it possible to provide more powerful REPL - completion. It allows side effects that are not noticable with the default - parser structure to still be completeable. - - The biggest difference from CompiledObject to MixedObject is that we are - generally dealing with Python code and not with C code. This will generate - fewer special cases, because we in Python you don't have the same freedoms - to modify the runtime. - """ - def __init__(self, evaluator, obj, node_name): - self._evaluator = evaluator - self.obj = obj - self.node_name = node_name - self.definition = node_name.get_definition() - - @property - def names_dict(self): - return LazyMixedNamesDict(self._evaluator, self) - - def names_dicts(self, search_global): - # TODO is this needed? - assert search_global is False - return [self.names_dict] - - def api_type(self): - mappings = { - 'expr_stmt': 'statement', - 'classdef': 'class', - 'funcdef': 'function', - 'file_input': 'module', - } - return mappings[self.definition.type] - - def __repr__(self): - return '<%s: %s>' % (type(self).__name__, repr(self.obj)) - - def __getattr__(self, name): - return getattr(self.definition, name) - - -class MixedName(compiled.CompiledName): - """ - The ``CompiledName._compiled_object`` is our MixedObject. - """ - @property - @underscore_memoization - def parent(self): - return create(self._evaluator, getattr(self._compiled_obj.obj, self.name)) - - @parent.setter - def parent(self, value): - pass # Just ignore this, Name tries to overwrite the parent attribute. - - @property - def start_pos(self): - if isinstance(self.parent, MixedObject): - return self.parent.node_name.start_pos - - # This means a start_pos that doesn't exist (compiled objects). - return (0, 0) - - -class LazyMixedNamesDict(compiled.LazyNamesDict): - name_class = MixedName - - -def parse(grammar, path): - with open(path) as f: - source = f.read() - source = common.source_to_unicode(source) - return FastParser(grammar, source, path) - - -def _load_module(evaluator, path, python_object): - module = parse(evaluator.grammar, path).module - python_module = inspect.getmodule(python_object) - - evaluator.modules[python_module.__name__] = module - return module - - -def find_syntax_node_name(evaluator, python_object): - try: - path = inspect.getsourcefile(python_object) - except TypeError: - # The type might not be known (e.g. class_with_dict.__weakref__) - return None - if path is None or not os.path.exists(path): - # The path might not exist or be e.g. . - return None - - module = _load_module(evaluator, path, python_object) - - if inspect.ismodule(python_object): - # We don't need to check names for modules, because there's not really - # a way to write a module in a module in Python (and also __name__ can - # be something like ``email.utils``). - return module - - name_str = python_object.__name__ - if name_str == '': - return None # It's too hard to find lambdas. - - names = module.used_names[name_str] - names = [n for n in names if n.is_definition()] - - try: - code = python_object.__code__ - # By using the line number of a code object we make the lookup in a - # file pretty easy. There's still a possibility of people defining - # stuff like ``a = 3; foo(a); a = 4`` on the same line, but if people - # do so we just don't care. - line_nr = code.co_firstlineno - except AttributeError: - pass - else: - line_names = [name for name in names if name.start_pos[0] == line_nr] - # There's a chance that the object is not available anymore, because - # the code has changed in the background. - if line_names: - return line_names[-1] - - # It's really hard to actually get the right definition, here as a last - # resort we just return the last one. This chance might lead to odd - # completions at some points but will lead to mostly correct type - # inference, because people tend to define a public name in a module only - # once. - return names[-1] - - -@compiled.compiled_objects_cache('mixed_cache') -def create(evaluator, obj): - name = find_syntax_node_name(evaluator, obj) - if name is None: - return compiled.create(evaluator, obj) - else: - return MixedObject(evaluator, obj, name) diff --git a/pythonFiles/preview/jedi/evaluate/docstrings.py b/pythonFiles/preview/jedi/evaluate/docstrings.py deleted file mode 100644 index d2ab34ed10c6..000000000000 --- a/pythonFiles/preview/jedi/evaluate/docstrings.py +++ /dev/null @@ -1,204 +0,0 @@ -""" -Docstrings are another source of information for functions and classes. -:mod:`jedi.evaluate.dynamic` tries to find all executions of functions, while -the docstring parsing is much easier. There are two different types of -docstrings that |jedi| understands: - -- `Sphinx `_ -- `Epydoc `_ - -For example, the sphinx annotation ``:type foo: str`` clearly states that the -type of ``foo`` is ``str``. - -As an addition to parameter searching, this module also provides return -annotations. -""" - -from ast import literal_eval -import re -from itertools import chain -from textwrap import dedent - -from jedi.evaluate.cache import memoize_default -from jedi.parser import ParserWithRecovery, load_grammar -from jedi.parser.tree import Class -from jedi.common import indent_block -from jedi.evaluate.iterable import Array, FakeSequence, AlreadyEvaluated - - -DOCSTRING_PARAM_PATTERNS = [ - r'\s*:type\s+%s:\s*([^\n]+)', # Sphinx - r'\s*:param\s+(\w+)\s+%s:[^\n]+', # Sphinx param with type - r'\s*@type\s+%s:\s*([^\n]+)', # Epydoc -] - -DOCSTRING_RETURN_PATTERNS = [ - re.compile(r'\s*:rtype:\s*([^\n]+)', re.M), # Sphinx - re.compile(r'\s*@rtype:\s*([^\n]+)', re.M), # Epydoc -] - -REST_ROLE_PATTERN = re.compile(r':[^`]+:`([^`]+)`') - - -try: - from numpydoc.docscrape import NumpyDocString -except ImportError: - def _search_param_in_numpydocstr(docstr, param_str): - return [] -else: - def _search_param_in_numpydocstr(docstr, param_str): - """Search `docstr` (in numpydoc format) for type(-s) of `param_str`.""" - params = NumpyDocString(docstr)._parsed_data['Parameters'] - for p_name, p_type, p_descr in params: - if p_name == param_str: - m = re.match('([^,]+(,[^,]+)*?)(,[ ]*optional)?$', p_type) - if m: - p_type = m.group(1) - - if p_type.startswith('{'): - types = set(type(x).__name__ for x in literal_eval(p_type)) - return list(types) - else: - return [p_type] - return [] - - -def _search_param_in_docstr(docstr, param_str): - """ - Search `docstr` for type(-s) of `param_str`. - - >>> _search_param_in_docstr(':type param: int', 'param') - ['int'] - >>> _search_param_in_docstr('@type param: int', 'param') - ['int'] - >>> _search_param_in_docstr( - ... ':type param: :class:`threading.Thread`', 'param') - ['threading.Thread'] - >>> bool(_search_param_in_docstr('no document', 'param')) - False - >>> _search_param_in_docstr(':param int param: some description', 'param') - ['int'] - - """ - # look at #40 to see definitions of those params - patterns = [re.compile(p % re.escape(param_str)) - for p in DOCSTRING_PARAM_PATTERNS] - for pattern in patterns: - match = pattern.search(docstr) - if match: - return [_strip_rst_role(match.group(1))] - - return (_search_param_in_numpydocstr(docstr, param_str) or - []) - - -def _strip_rst_role(type_str): - """ - Strip off the part looks like a ReST role in `type_str`. - - >>> _strip_rst_role(':class:`ClassName`') # strip off :class: - 'ClassName' - >>> _strip_rst_role(':py:obj:`module.Object`') # works with domain - 'module.Object' - >>> _strip_rst_role('ClassName') # do nothing when not ReST role - 'ClassName' - - See also: - http://sphinx-doc.org/domains.html#cross-referencing-python-objects - - """ - match = REST_ROLE_PATTERN.match(type_str) - if match: - return match.group(1) - else: - return type_str - - -def _evaluate_for_statement_string(evaluator, string, module): - code = dedent(""" - def pseudo_docstring_stuff(): - # Create a pseudo function for docstring statements. - %s - """) - if string is None: - return [] - - for element in re.findall('((?:\w+\.)*\w+)\.', string): - # Try to import module part in dotted name. - # (e.g., 'threading' in 'threading.Thread'). - string = 'import %s\n' % element + string - - # Take the default grammar here, if we load the Python 2.7 grammar here, it - # will be impossible to use `...` (Ellipsis) as a token. Docstring types - # don't need to conform with the current grammar. - p = ParserWithRecovery(load_grammar(), code % indent_block(string)) - try: - pseudo_cls = p.module.subscopes[0] - # First pick suite, then simple_stmt (-2 for DEDENT) and then the node, - # which is also not the last item, because there's a newline. - stmt = pseudo_cls.children[-1].children[-2].children[-2] - except (AttributeError, IndexError): - return [] - - # Use the module of the param. - # TODO this module is not the module of the param in case of a function - # call. In that case it's the module of the function call. - # stuffed with content from a function call. - pseudo_cls.parent = module - return list(_execute_types_in_stmt(evaluator, stmt)) - - -def _execute_types_in_stmt(evaluator, stmt): - """ - Executing all types or general elements that we find in a statement. This - doesn't include tuple, list and dict literals, because the stuff they - contain is executed. (Used as type information). - """ - definitions = evaluator.eval_element(stmt) - return chain.from_iterable(_execute_array_values(evaluator, d) for d in definitions) - - -def _execute_array_values(evaluator, array): - """ - Tuples indicate that there's not just one return value, but the listed - ones. `(str, int)` means that it returns a tuple with both types. - """ - if isinstance(array, Array): - values = [] - for types in array.py__iter__(): - objects = set(chain.from_iterable(_execute_array_values(evaluator, typ) for typ in types)) - values.append(AlreadyEvaluated(objects)) - return [FakeSequence(evaluator, values, array.type)] - else: - return evaluator.execute(array) - - -@memoize_default(None, evaluator_is_first_arg=True) -def follow_param(evaluator, param): - def eval_docstring(docstring): - return set( - [p for param_str in _search_param_in_docstr(docstring, str(param.name)) - for p in _evaluate_for_statement_string(evaluator, param_str, module)] - ) - func = param.parent_function - module = param.get_parent_until() - - types = eval_docstring(func.raw_doc) - if func.name.value == '__init__': - cls = func.get_parent_until(Class) - if cls.type == 'classdef': - types |= eval_docstring(cls.raw_doc) - - return types - - -@memoize_default(None, evaluator_is_first_arg=True) -def find_return_types(evaluator, func): - def search_return_in_docstr(code): - for p in DOCSTRING_RETURN_PATTERNS: - match = p.search(code) - if match: - return _strip_rst_role(match.group(1)) - - type_str = search_return_in_docstr(func.raw_doc) - return _evaluate_for_statement_string(evaluator, type_str, func.get_parent_until()) diff --git a/pythonFiles/preview/jedi/evaluate/dynamic.py b/pythonFiles/preview/jedi/evaluate/dynamic.py deleted file mode 100644 index d0570b59a7c5..000000000000 --- a/pythonFiles/preview/jedi/evaluate/dynamic.py +++ /dev/null @@ -1,149 +0,0 @@ -""" -One of the really important features of |jedi| is to have an option to -understand code like this:: - - def foo(bar): - bar. # completion here - foo(1) - -There's no doubt wheter bar is an ``int`` or not, but if there's also a call -like ``foo('str')``, what would happen? Well, we'll just show both. Because -that's what a human would expect. - -It works as follows: - -- |Jedi| sees a param -- search for function calls named ``foo`` -- execute these calls and check the input. This work with a ``ParamListener``. -""" -from itertools import chain - -from jedi._compatibility import unicode -from jedi.parser import tree -from jedi import settings -from jedi import debug -from jedi.evaluate.cache import memoize_default -from jedi.evaluate import imports - - -MAX_PARAM_SEARCHES = 20 - - -class ParamListener(object): - """ - This listener is used to get the params for a function. - """ - def __init__(self): - self.param_possibilities = [] - - def execute(self, params): - self.param_possibilities += params - - -@debug.increase_indent -def search_params(evaluator, param): - """ - A dynamic search for param values. If you try to complete a type: - - >>> def func(foo): - ... foo - >>> func(1) - >>> func("") - - It is not known what the type ``foo`` without analysing the whole code. You - have to look for all calls to ``func`` to find out what ``foo`` possibly - is. - """ - if not settings.dynamic_params: - return set() - - evaluator.dynamic_params_depth += 1 - try: - func = param.get_parent_until(tree.Function) - debug.dbg('Dynamic param search for %s in %s.', param, str(func.name), color='MAGENTA') - # Compare the param names. - names = [n for n in search_function_call(evaluator, func) - if n.value == param.name.value] - # Evaluate the ExecutedParams to types. - result = set(chain.from_iterable(n.parent.eval(evaluator) for n in names)) - debug.dbg('Dynamic param result %s', result, color='MAGENTA') - return result - finally: - evaluator.dynamic_params_depth -= 1 - - -@memoize_default([], evaluator_is_first_arg=True) -def search_function_call(evaluator, func): - """ - Returns a list of param names. - """ - from jedi.evaluate import representation as er - - def get_possible_nodes(module, func_name): - try: - names = module.used_names[func_name] - except KeyError: - return - - for name in names: - bracket = name.get_next_leaf() - trailer = bracket.parent - if trailer.type == 'trailer' and bracket == '(': - yield name, trailer - - def undecorate(typ): - # We have to remove decorators, because they are not the - # "original" functions, this way we can easily compare. - # At the same time we also have to remove InstanceElements. - if typ.isinstance(er.Function, er.Instance) \ - and typ.decorates is not None: - return typ.decorates - elif isinstance(typ, er.InstanceElement): - return typ.var - else: - return typ - - current_module = func.get_parent_until() - func_name = unicode(func.name) - compare = func - if func_name == '__init__': - cls = func.get_parent_scope() - if isinstance(cls, tree.Class): - func_name = unicode(cls.name) - compare = cls - - # add the listener - listener = ParamListener() - func.listeners.add(listener) - - try: - result = [] - i = 0 - for mod in imports.get_modules_containing_name(evaluator, [current_module], func_name): - for name, trailer in get_possible_nodes(mod, func_name): - i += 1 - - # This is a simple way to stop Jedi's dynamic param recursion - # from going wild: The deeper Jedi's in the recursin, the less - # code should be evaluated. - if i * evaluator.dynamic_params_depth > MAX_PARAM_SEARCHES: - return listener.param_possibilities - - for typ in evaluator.goto_definitions(name): - undecorated = undecorate(typ) - if evaluator.wrap(compare) == undecorated: - # Only if we have the correct function we execute - # it, otherwise just ignore it. - evaluator.eval_trailer([typ], trailer) - - result = listener.param_possibilities - - # If there are results after processing a module, we're probably - # good to process. - if result: - return result - finally: - # cleanup: remove the listener; important: should not stick. - func.listeners.remove(listener) - - return set() diff --git a/pythonFiles/preview/jedi/evaluate/finder.py b/pythonFiles/preview/jedi/evaluate/finder.py deleted file mode 100644 index 2095959e93aa..000000000000 --- a/pythonFiles/preview/jedi/evaluate/finder.py +++ /dev/null @@ -1,632 +0,0 @@ -""" -Searching for names with given scope and name. This is very central in Jedi and -Python. The name resolution is quite complicated with descripter, -``__getattribute__``, ``__getattr__``, ``global``, etc. - -If you want to understand name resolution, please read the first few chapters -in http://blog.ionelmc.ro/2015/02/09/understanding-python-metaclasses/. - -Flow checks -+++++++++++ - -Flow checks are not really mature. There's only a check for ``isinstance``. It -would check whether a flow has the form of ``if isinstance(a, type_or_tuple)``. -Unfortunately every other thing is being ignored (e.g. a == '' would be easy to -check for -> a is a string). There's big potential in these checks. -""" -from itertools import chain - -from jedi._compatibility import unicode -from jedi.parser import tree -from jedi import debug -from jedi import common -from jedi.common import unite -from jedi import settings -from jedi.evaluate import representation as er -from jedi.evaluate import dynamic -from jedi.evaluate import compiled -from jedi.evaluate import docstrings -from jedi.evaluate import pep0484 -from jedi.evaluate import iterable -from jedi.evaluate import imports -from jedi.evaluate import analysis -from jedi.evaluate import flow_analysis -from jedi.evaluate import param -from jedi.evaluate import helpers -from jedi.evaluate.cache import memoize_default - - -def filter_after_position(names, position): - """ - Removes all names after a certain position. If position is None, just - returns the names list. - """ - if position is None: - return names - - names_new = [] - for n in names: - # Filter positions and also allow list comprehensions and lambdas. - if n.start_pos[0] is not None and n.start_pos < position \ - or isinstance(n.get_definition(), (tree.CompFor, tree.Lambda)): - names_new.append(n) - return names_new - - -def filter_definition_names(names, origin, position=None): - """ - Filter names that are actual definitions in a scope. Names that are just - used will be ignored. - """ - if not names: - return [] - - # Just calculate the scope from the first - stmt = names[0].get_definition() - scope = stmt.get_parent_scope() - - if not (isinstance(scope, er.FunctionExecution) and - isinstance(scope.base, er.LambdaWrapper)): - names = filter_after_position(names, position) - names = [name for name in names if name.is_definition()] - - # Private name mangling (compile.c) disallows access on names - # preceeded by two underscores `__` if used outside of the class. Names - # that also end with two underscores (e.g. __id__) are not affected. - for name in list(names): - if name.value.startswith('__') and not name.value.endswith('__'): - if filter_private_variable(scope, origin): - names.remove(name) - return names - - -class NameFinder(object): - def __init__(self, evaluator, scope, name_str, position=None): - self._evaluator = evaluator - # Make sure that it's not just a syntax tree node. - self.scope = evaluator.wrap(scope) - self.name_str = name_str - self.position = position - self._found_predefined_if_name = None - - @debug.increase_indent - def find(self, scopes, attribute_lookup): - """ - :params bool attribute_lookup: Tell to logic if we're accessing the - attribute or the contents of e.g. a function. - """ - # TODO rename scopes to names_dicts - - names = self.filter_name(scopes) - if self._found_predefined_if_name is not None: - return self._found_predefined_if_name - - types = self._names_to_types(names, attribute_lookup) - - if not names and not types \ - and not (isinstance(self.name_str, tree.Name) and - isinstance(self.name_str.parent.parent, tree.Param)): - if not isinstance(self.name_str, (str, unicode)): # TODO Remove? - if attribute_lookup: - analysis.add_attribute_error(self._evaluator, - self.scope, self.name_str) - else: - message = ("NameError: name '%s' is not defined." - % self.name_str) - analysis.add(self._evaluator, 'name-error', self.name_str, - message) - - debug.dbg('finder._names_to_types: %s -> %s', names, types) - return types - - def scopes(self, search_global=False): - if search_global: - return global_names_dict_generator(self._evaluator, self.scope, self.position) - else: - return ((n, None) for n in self.scope.names_dicts(search_global)) - - def names_dict_lookup(self, names_dict, position): - def get_param(scope, el): - if isinstance(el.get_parent_until(tree.Param), tree.Param): - return scope.param_by_name(str(el)) - return el - - search_str = str(self.name_str) - try: - names = names_dict[search_str] - if not names: # We want names, otherwise stop. - return [] - except KeyError: - return [] - - names = filter_definition_names(names, self.name_str, position) - - name_scope = None - # Only the names defined in the last position are valid definitions. - last_names = [] - for name in reversed(sorted(names, key=lambda name: name.start_pos)): - stmt = name.get_definition() - name_scope = self._evaluator.wrap(stmt.get_parent_scope()) - - if isinstance(self.scope, er.Instance) and not isinstance(name_scope, er.Instance): - # Instances should not be checked for positioning, because we - # don't know in which order the functions are called. - last_names.append(name) - continue - - if isinstance(name_scope, compiled.CompiledObject): - # Let's test this. TODO need comment. shouldn't this be - # filtered before? - last_names.append(name) - continue - - if isinstance(stmt, er.ModuleWrapper): - # In case of REPL completion, we can infer modules names that - # don't really have a definition (because they are really just - # namespaces). In this case we can just add it. - last_names.append(name) - continue - - if isinstance(name, compiled.CompiledName) \ - or isinstance(name, er.InstanceName) and isinstance(name._origin_name, compiled.CompiledName): - last_names.append(name) - continue - - if isinstance(self.name_str, tree.Name): - origin_scope = self.name_str.get_parent_until(tree.Scope, reverse=True) - scope = self.name_str - check = None - while True: - scope = scope.parent - if scope.type in ("if_stmt", "for_stmt", "comp_for"): - try: - name_dict = self._evaluator.predefined_if_name_dict_dict[scope] - types = set(name_dict[str(self.name_str)]) - except KeyError: - continue - else: - if self.name_str.start_pos < scope.children[1].end_pos: - # It doesn't make any sense to check if - # statements in the if statement itself, just - # deliver types. - self._found_predefined_if_name = types - else: - check = flow_analysis.break_check(self._evaluator, self.scope, - origin_scope) - if check is flow_analysis.UNREACHABLE: - self._found_predefined_if_name = set() - else: - self._found_predefined_if_name = types - break - if isinstance(scope, tree.IsScope) or scope is None: - break - else: - origin_scope = None - - if isinstance(stmt.parent, compiled.CompiledObject): - # TODO seriously? this is stupid. - continue - check = flow_analysis.break_check(self._evaluator, name_scope, - stmt, origin_scope) - if check is not flow_analysis.UNREACHABLE: - last_names.append(name) - - if check is flow_analysis.REACHABLE: - break - - if isinstance(name_scope, er.FunctionExecution): - # Replace params - return [get_param(name_scope, n) for n in last_names] - return last_names - - def filter_name(self, names_dicts): - """ - Searches names that are defined in a scope (the different - `names_dicts`), until a name fits. - """ - names = [] - for names_dict, position in names_dicts: - names = self.names_dict_lookup(names_dict, position) - if names: - break - - debug.dbg('finder.filter_name "%s" in (%s): %s@%s', self.name_str, - self.scope, names, self.position) - return list(self._clean_names(names)) - - def _clean_names(self, names): - """ - ``NameFinder.filter_name`` should only output names with correct - wrapper parents. We don't want to see AST classes out in the - evaluation, so remove them already here! - """ - for n in names: - definition = n.parent - if isinstance(definition, (compiled.CompiledObject, - iterable.BuiltinMethod)): - # TODO this if should really be removed by changing the type of - # those classes. - yield n - elif definition.type in ('funcdef', 'classdef', 'file_input'): - yield self._evaluator.wrap(definition).name - else: - yield n - - def _check_getattr(self, inst): - """Checks for both __getattr__ and __getattribute__ methods""" - result = set() - # str is important, because it shouldn't be `Name`! - name = compiled.create(self._evaluator, str(self.name_str)) - with common.ignored(KeyError): - result = inst.execute_subscope_by_name('__getattr__', name) - if not result: - # This is a little bit special. `__getattribute__` is in Python - # executed before `__getattr__`. But: I know no use case, where - # this could be practical and where jedi would return wrong types. - # If you ever find something, let me know! - # We are inversing this, because a hand-crafted `__getattribute__` - # could still call another hand-crafted `__getattr__`, but not the - # other way around. - with common.ignored(KeyError): - result = inst.execute_subscope_by_name('__getattribute__', name) - return result - - def _names_to_types(self, names, attribute_lookup): - types = set() - - # Add isinstance and other if/assert knowledge. - if isinstance(self.name_str, tree.Name): - # Ignore FunctionExecution parents for now. - flow_scope = self.name_str - until = flow_scope.get_parent_until(er.FunctionExecution) - while not isinstance(until, er.FunctionExecution): - flow_scope = flow_scope.get_parent_scope(include_flows=True) - if flow_scope is None: - break - # TODO check if result is in scope -> no evaluation necessary - n = check_flow_information(self._evaluator, flow_scope, - self.name_str, self.position) - if n: - return n - - for name in names: - new_types = _name_to_types(self._evaluator, name, self.scope) - if isinstance(self.scope, (er.Class, er.Instance)) and attribute_lookup: - types |= set(self._resolve_descriptors(name, new_types)) - else: - types |= set(new_types) - if not names and isinstance(self.scope, er.Instance): - # handling __getattr__ / __getattribute__ - return self._check_getattr(self.scope) - - return types - - def _resolve_descriptors(self, name, types): - # The name must not be in the dictionary, but part of the class - # definition. __get__ is only called if the descriptor is defined in - # the class dictionary. - name_scope = name.get_definition().get_parent_scope() - if not isinstance(name_scope, (er.Instance, tree.Class)): - return types - - result = set() - for r in types: - try: - desc_return = r.get_descriptor_returns - except AttributeError: - result.add(r) - else: - result |= desc_return(self.scope) - return result - - -def _get_global_stmt_scopes(evaluator, global_stmt, name): - global_stmt_scope = global_stmt.get_parent_scope() - module = global_stmt_scope.get_parent_until() - for used_name in module.used_names[str(name)]: - if used_name.parent.type == 'global_stmt': - yield evaluator.wrap(used_name.get_parent_scope()) - - -@memoize_default(set(), evaluator_is_first_arg=True) -def _name_to_types(evaluator, name, scope): - types = [] - typ = name.get_definition() - if typ.isinstance(tree.ForStmt): - types = pep0484.find_type_from_comment_hint_for(evaluator, typ, name) - if types: - return types - if typ.isinstance(tree.WithStmt): - types = pep0484.find_type_from_comment_hint_with(evaluator, typ, name) - if types: - return types - if typ.isinstance(tree.ForStmt, tree.CompFor): - container_types = evaluator.eval_element(typ.children[3]) - for_types = iterable.py__iter__types(evaluator, container_types, typ.children[3]) - types = check_tuple_assignments(evaluator, for_types, name) - elif isinstance(typ, tree.Param): - types = _eval_param(evaluator, typ, scope) - elif typ.isinstance(tree.ExprStmt): - types = _remove_statements(evaluator, typ, name) - elif typ.isinstance(tree.WithStmt): - types = evaluator.eval_element(typ.node_from_name(name)) - elif isinstance(typ, tree.Import): - types = imports.ImportWrapper(evaluator, name).follow() - elif typ.type == 'global_stmt': - for s in _get_global_stmt_scopes(evaluator, typ, name): - finder = NameFinder(evaluator, s, str(name)) - names_dicts = finder.scopes(search_global=True) - # For global_stmt lookups, we only need the first possible scope, - # which means the function itself. - names_dicts = [next(names_dicts)] - types += finder.find(names_dicts, attribute_lookup=False) - elif isinstance(typ, tree.TryStmt): - # TODO an exception can also be a tuple. Check for those. - # TODO check for types that are not classes and add it to - # the static analysis report. - exceptions = evaluator.eval_element(name.get_previous_sibling().get_previous_sibling()) - types = set(chain.from_iterable(evaluator.execute(t) for t in exceptions)) - else: - if typ.isinstance(er.Function): - typ = typ.get_decorated_func() - types = set([typ]) - return types - - -def _remove_statements(evaluator, stmt, name): - """ - This is the part where statements are being stripped. - - Due to lazy evaluation, statements like a = func; b = a; b() have to be - evaluated. - """ - types = set() - # Remove the statement docstr stuff for now, that has to be - # implemented with the evaluator class. - #if stmt.docstr: - #res_new.append(stmt) - - check_instance = None - if isinstance(stmt, er.InstanceElement) and stmt.is_class_var: - check_instance = stmt.instance - stmt = stmt.var - - pep0484types = \ - pep0484.find_type_from_comment_hint_assign(evaluator, stmt, name) - if pep0484types: - return pep0484types - types |= evaluator.eval_statement(stmt, seek_name=name) - - if check_instance is not None: - # class renames - types = set([er.get_instance_el(evaluator, check_instance, a, True) - if isinstance(a, (er.Function, tree.Function)) - else a for a in types]) - return types - - -def _eval_param(evaluator, param, scope): - res_new = set() - func = param.get_parent_scope() - - cls = func.parent.get_parent_until((tree.Class, tree.Function)) - - from jedi.evaluate.param import ExecutedParam, Arguments - if isinstance(cls, tree.Class) and param.position_nr == 0 \ - and not isinstance(param, ExecutedParam): - # This is where we add self - if it has never been - # instantiated. - if isinstance(scope, er.InstanceElement): - res_new.add(scope.instance) - else: - inst = er.Instance(evaluator, evaluator.wrap(cls), - Arguments(evaluator, ()), is_generated=True) - res_new.add(inst) - return res_new - - # Instances are typically faked, if the instance is not called from - # outside. Here we check it for __init__ functions and return. - if isinstance(func, er.InstanceElement) \ - and func.instance.is_generated and str(func.name) == '__init__': - param = func.var.params[param.position_nr] - - # Add pep0484 and docstring knowledge. - pep0484_hints = pep0484.follow_param(evaluator, param) - doc_params = docstrings.follow_param(evaluator, param) - if pep0484_hints or doc_params: - return list(set(pep0484_hints) | set(doc_params)) - - if isinstance(param, ExecutedParam): - return res_new | param.eval(evaluator) - else: - # Param owns no information itself. - res_new |= dynamic.search_params(evaluator, param) - if not res_new: - if param.stars: - t = 'tuple' if param.stars == 1 else 'dict' - typ = list(evaluator.find_types(evaluator.BUILTINS, t))[0] - res_new = evaluator.execute(typ) - if param.default: - res_new |= evaluator.eval_element(param.default) - return res_new - - -def check_flow_information(evaluator, flow, search_name, pos): - """ Try to find out the type of a variable just with the information that - is given by the flows: e.g. It is also responsible for assert checks.:: - - if isinstance(k, str): - k. # <- completion here - - ensures that `k` is a string. - """ - if not settings.dynamic_flow_information: - return None - - result = set() - if flow.is_scope(): - # Check for asserts. - try: - names = reversed(flow.names_dict[search_name.value]) - except (KeyError, AttributeError): - names = [] - - for name in names: - ass = name.get_parent_until(tree.AssertStmt) - if isinstance(ass, tree.AssertStmt) and pos is not None and ass.start_pos < pos: - result = _check_isinstance_type(evaluator, ass.assertion(), search_name) - if result: - break - - if isinstance(flow, (tree.IfStmt, tree.WhileStmt)): - potential_ifs = [c for c in flow.children[1::4] if c != ':'] - for if_test in reversed(potential_ifs): - if search_name.start_pos > if_test.end_pos: - return _check_isinstance_type(evaluator, if_test, search_name) - return result - - -def _check_isinstance_type(evaluator, element, search_name): - try: - assert element.type in ('power', 'atom_expr') - # this might be removed if we analyze and, etc - assert len(element.children) == 2 - first, trailer = element.children - assert isinstance(first, tree.Name) and first.value == 'isinstance' - assert trailer.type == 'trailer' and trailer.children[0] == '(' - assert len(trailer.children) == 3 - - # arglist stuff - arglist = trailer.children[1] - args = param.Arguments(evaluator, arglist, trailer) - lst = list(args.unpack()) - # Disallow keyword arguments - assert len(lst) == 2 and lst[0][0] is None and lst[1][0] is None - name = lst[0][1][0] # first argument, values, first value - # Do a simple get_code comparison. They should just have the same code, - # and everything will be all right. - classes = lst[1][1][0] - call = helpers.call_of_leaf(search_name) - assert name.get_code(normalized=True) == call.get_code(normalized=True) - except AssertionError: - return set() - - result = set() - for cls_or_tup in evaluator.eval_element(classes): - if isinstance(cls_or_tup, iterable.Array) and cls_or_tup.type == 'tuple': - for typ in unite(cls_or_tup.py__iter__()): - result |= evaluator.execute(typ) - else: - result |= evaluator.execute(cls_or_tup) - return result - - -def global_names_dict_generator(evaluator, scope, position): - """ - For global name lookups. Yields tuples of (names_dict, position). If the - position is None, the position does not matter anymore in that scope. - - This function is used to include names from outer scopes. For example, when - the current scope is function: - - >>> from jedi._compatibility import u, no_unicode_pprint - >>> from jedi.parser import ParserWithRecovery, load_grammar - >>> parser = ParserWithRecovery(load_grammar(), u(''' - ... x = ['a', 'b', 'c'] - ... def func(): - ... y = None - ... ''')) - >>> scope = parser.module.subscopes[0] - >>> scope - - - `global_names_dict_generator` is a generator. First it yields names from - most inner scope. - - >>> from jedi.evaluate import Evaluator - >>> evaluator = Evaluator(load_grammar()) - >>> scope = evaluator.wrap(scope) - >>> pairs = list(global_names_dict_generator(evaluator, scope, (4, 0))) - >>> no_unicode_pprint(pairs[0]) - ({'func': [], 'y': []}, (4, 0)) - - Then it yields the names from one level "lower". In this example, this - is the most outer scope. As you can see, the position in the tuple is now - None, because typically the whole module is loaded before the function is - called. - - >>> no_unicode_pprint(pairs[1]) - ({'func': [], 'x': []}, None) - - After that we have a few underscore names that are part of the module. - - >>> sorted(pairs[2][0].keys()) - ['__doc__', '__file__', '__name__', '__package__'] - >>> pairs[3] # global names -> there are none in our example. - ({}, None) - >>> pairs[4] # package modules -> Also none. - ({}, None) - - Finally, it yields names from builtin, if `include_builtin` is - true (default). - - >>> pairs[5][0].values() #doctest: +ELLIPSIS - [[], ...] - """ - in_func = False - while scope is not None: - if not (scope.type == 'classdef' and in_func): - # Names in methods cannot be resolved within the class. - - for names_dict in scope.names_dicts(True): - yield names_dict, position - if hasattr(scope, 'resets_positions'): - # TODO This is so ugly, seriously. However there's - # currently no good way of influencing - # global_names_dict_generator when it comes to certain - # objects. - position = None - if scope.type == 'funcdef': - # The position should be reset if the current scope is a function. - in_func = True - position = None - scope = evaluator.wrap(scope.get_parent_scope()) - - # Add builtins to the global scope. - for names_dict in evaluator.BUILTINS.names_dicts(True): - yield names_dict, None - - -def check_tuple_assignments(evaluator, types, name): - """ - Checks if tuples are assigned. - """ - for index, node in name.assignment_indexes(): - iterated = iterable.py__iter__(evaluator, types, node) - for _ in range(index + 1): - try: - types = next(iterated) - except StopIteration: - # We could do this with the default param in next. But this - # would allow this loop to run for a very long time if the - # index number is high. Therefore break if the loop is - # finished. - types = set() - break - return types - - -def filter_private_variable(scope, origin_node): - """Check if a variable is defined inside the same class or outside.""" - instance = scope.get_parent_scope() - coming_from = origin_node - while coming_from is not None \ - and not isinstance(coming_from, (tree.Class, compiled.CompiledObject)): - coming_from = coming_from.get_parent_scope() - - # CompiledObjects don't have double underscore attributes, but Jedi abuses - # those for fakes (builtins.pym -> list). - if isinstance(instance, compiled.CompiledObject): - return instance != coming_from - else: - return isinstance(instance, er.Instance) and instance.base.base != coming_from diff --git a/pythonFiles/preview/jedi/evaluate/flow_analysis.py b/pythonFiles/preview/jedi/evaluate/flow_analysis.py deleted file mode 100644 index e188264bc0d8..000000000000 --- a/pythonFiles/preview/jedi/evaluate/flow_analysis.py +++ /dev/null @@ -1,91 +0,0 @@ -from jedi.parser import tree - - -class Status(object): - lookup_table = {} - - def __init__(self, value, name): - self._value = value - self._name = name - Status.lookup_table[value] = self - - def invert(self): - if self is REACHABLE: - return UNREACHABLE - elif self is UNREACHABLE: - return REACHABLE - else: - return UNSURE - - def __and__(self, other): - if UNSURE in (self, other): - return UNSURE - else: - return REACHABLE if self._value and other._value else UNREACHABLE - - def __repr__(self): - return '<%s: %s>' % (type(self).__name__, self._name) - - -REACHABLE = Status(True, 'reachable') -UNREACHABLE = Status(False, 'unreachable') -UNSURE = Status(None, 'unsure') - - -def break_check(evaluator, base_scope, stmt, origin_scope=None): - element_scope = evaluator.wrap(stmt.get_parent_scope(include_flows=True)) - # Direct parents get resolved, we filter scopes that are separate branches. - # This makes sense for autocompletion and static analysis. For actual - # Python it doesn't matter, because we're talking about potentially - # unreachable code. - # e.g. `if 0:` would cause all name lookup within the flow make - # unaccessible. This is not a "problem" in Python, because the code is - # never called. In Jedi though, we still want to infer types. - while origin_scope is not None: - if element_scope == origin_scope: - return REACHABLE - origin_scope = origin_scope.parent - x = _break_check(evaluator, stmt, base_scope, element_scope) - return x - - -def _break_check(evaluator, stmt, base_scope, element_scope): - element_scope = evaluator.wrap(element_scope) - base_scope = evaluator.wrap(base_scope) - - reachable = REACHABLE - if isinstance(element_scope, tree.IfStmt): - if element_scope.node_after_else(stmt): - for check_node in element_scope.check_nodes(): - reachable = _check_if(evaluator, check_node) - if reachable in (REACHABLE, UNSURE): - break - reachable = reachable.invert() - else: - node = element_scope.node_in_which_check_node(stmt) - if node is not None: - reachable = _check_if(evaluator, node) - elif isinstance(element_scope, (tree.TryStmt, tree.WhileStmt)): - return UNSURE - - # Only reachable branches need to be examined further. - if reachable in (UNREACHABLE, UNSURE): - return reachable - - if element_scope.type == 'file_input': - # The definition is in another module and therefore just return what we - # have generated. - return reachable - if base_scope != element_scope and base_scope != element_scope.parent: - return reachable & _break_check(evaluator, stmt, base_scope, element_scope.parent) - else: - return reachable - - -def _check_if(evaluator, node): - types = evaluator.eval_element(node) - values = set(x.py__bool__() for x in types) - if len(values) == 1: - return Status.lookup_table[values.pop()] - else: - return UNSURE diff --git a/pythonFiles/preview/jedi/evaluate/helpers.py b/pythonFiles/preview/jedi/evaluate/helpers.py deleted file mode 100644 index 27293edaf875..000000000000 --- a/pythonFiles/preview/jedi/evaluate/helpers.py +++ /dev/null @@ -1,200 +0,0 @@ -import copy -from itertools import chain - -from jedi.parser import tree - - -def deep_ast_copy(obj, parent=None, new_elements=None): - """ - Much, much faster than copy.deepcopy, but just for Parser elements (Doesn't - copy parents). - """ - - if new_elements is None: - new_elements = {} - - def copy_node(obj): - # If it's already in the cache, just return it. - try: - return new_elements[obj] - except KeyError: - # Actually copy and set attributes. - new_obj = copy.copy(obj) - new_elements[obj] = new_obj - - # Copy children - new_children = [] - for child in obj.children: - typ = child.type - if typ in ('newline', 'operator', 'keyword', 'number', 'string', - 'indent', 'dedent', 'endmarker', 'error_leaf'): - # At the moment we're not actually copying those primitive - # elements, because there's really no need to. The parents are - # obviously wrong, but that's not an issue. - new_child = child - elif typ == 'name': - new_elements[child] = new_child = copy.copy(child) - new_child.parent = new_obj - else: # Is a BaseNode. - new_child = copy_node(child) - new_child.parent = new_obj - new_children.append(new_child) - new_obj.children = new_children - - # Copy the names_dict (if there is one). - try: - names_dict = obj.names_dict - except AttributeError: - pass - else: - try: - new_obj.names_dict = new_names_dict = {} - except AttributeError: # Impossible to set CompFor.names_dict - pass - else: - for string, names in names_dict.items(): - new_names_dict[string] = [new_elements[n] for n in names] - return new_obj - - if isinstance(obj, tree.BaseNode): - new_obj = copy_node(obj) - else: - # Special case of a Name object. - new_elements[obj] = new_obj = copy.copy(obj) - - if parent is not None: - new_obj.parent = parent - return new_obj - - -def call_of_leaf(leaf, cut_own_trailer=False): - """ - Creates a "call" node that consist of all ``trailer`` and ``power`` - objects. E.g. if you call it with ``append``:: - - list([]).append(3) or None - - You would get a node with the content ``list([]).append`` back. - - This generates a copy of the original ast node. - - If you're using the leaf, e.g. the bracket `)` it will return ``list([])``. - - # TODO remove cut_own_trailer option, since its always used with it. Just - # ignore it, It's not what we want anyway. Or document it better? - """ - trailer = leaf.parent - # The leaf may not be the last or first child, because there exist three - # different trailers: `( x )`, `[ x ]` and `.x`. In the first two examples - # we should not match anything more than x. - if trailer.type != 'trailer' or leaf not in (trailer.children[0], trailer.children[-1]): - if trailer.type == 'atom': - return trailer - return leaf - - power = trailer.parent - index = power.children.index(trailer) - power = deep_ast_copy(power) - if cut_own_trailer: - cut = index - else: - cut = index + 1 - power.children[cut:] = [] - - if power.type == 'error_node': - start = index - while True: - start -= 1 - if power.children[start].type != 'trailer': - break - transformed = tree.Node('power', power.children[start:]) - transformed.parent = power.parent - return transformed - - return power - - -def get_names_of_node(node): - try: - children = node.children - except AttributeError: - if node.type == 'name': - return [node] - else: - return [] - else: - return list(chain.from_iterable(get_names_of_node(c) for c in children)) - - -def get_module_names(module, all_scopes): - """ - Returns a dictionary with name parts as keys and their call paths as - values. - """ - if all_scopes: - dct = module.used_names - else: - dct = module.names_dict - return chain.from_iterable(dct.values()) - - -class FakeImport(tree.ImportName): - def __init__(self, name, parent, level=0): - super(FakeImport, self).__init__([]) - self.parent = parent - self._level = level - self.name = name - - def get_defined_names(self): - return [self.name] - - def aliases(self): - return {} - - @property - def level(self): - return self._level - - @property - def start_pos(self): - return 0, 0 - - def paths(self): - return [[self.name]] - - def is_definition(self): - return True - - -class FakeName(tree.Name): - def __init__(self, name_str, parent=None, start_pos=(0, 0), is_definition=None): - """ - In case is_definition is defined (not None), that bool value will be - returned. - """ - super(FakeName, self).__init__(tree.zero_position_modifier, name_str, start_pos) - self.parent = parent - self._is_definition = is_definition - - def get_definition(self): - return self.parent - - def is_definition(self): - if self._is_definition is None: - return super(FakeName, self).is_definition() - else: - return self._is_definition - - -class LazyName(FakeName): - def __init__(self, name, parent_callback, is_definition=None): - super(LazyName, self).__init__(name, is_definition=is_definition) - self._parent_callback = parent_callback - - @property - def parent(self): - return self._parent_callback() - - @parent.setter - def parent(self, value): - pass # Do nothing, super classes can try to set the parent. diff --git a/pythonFiles/preview/jedi/evaluate/imports.py b/pythonFiles/preview/jedi/evaluate/imports.py deleted file mode 100644 index d65d897b5413..000000000000 --- a/pythonFiles/preview/jedi/evaluate/imports.py +++ /dev/null @@ -1,517 +0,0 @@ -""" -:mod:`jedi.evaluate.imports` is here to resolve import statements and return -the modules/classes/functions/whatever, which they stand for. However there's -not any actual importing done. This module is about finding modules in the -filesystem. This can be quite tricky sometimes, because Python imports are not -always that simple. - -This module uses imp for python up to 3.2 and importlib for python 3.3 on; the -correct implementation is delegated to _compatibility. - -This module also supports import autocompletion, which means to complete -statements like ``from datetim`` (curser at the end would return ``datetime``). -""" -import imp -import os -import pkgutil -import sys -from itertools import chain - -from jedi._compatibility import find_module, unicode -from jedi import common -from jedi import debug -from jedi.parser import fast -from jedi.parser import tree -from jedi.parser.utils import save_parser, load_parser, parser_cache -from jedi.evaluate import sys_path -from jedi.evaluate import helpers -from jedi import settings -from jedi.common import source_to_unicode -from jedi.evaluate import compiled -from jedi.evaluate import analysis -from jedi.evaluate.cache import memoize_default, NO_DEFAULT - - -def completion_names(evaluator, imp, pos): - name = imp.name_for_position(pos) - module = evaluator.wrap(imp.get_parent_until()) - if name is None: - level = 0 - for node in imp.children: - if node.end_pos <= pos: - if node in ('.', '...'): - level += len(node.value) - import_path = [] - else: - # Completion on an existing name. - - # The import path needs to be reduced by one, because we're completing. - import_path = imp.path_for_name(name)[:-1] - level = imp.level - - importer = Importer(evaluator, tuple(import_path), module, level) - if isinstance(imp, tree.ImportFrom): - c = imp.children - only_modules = c[c.index('import')].start_pos >= pos - else: - only_modules = True - return importer.completion_names(evaluator, only_modules) - - -class ImportWrapper(tree.Base): - def __init__(self, evaluator, name): - self._evaluator = evaluator - self._name = name - - self._import = name.get_parent_until(tree.Import) - self.import_path = self._import.path_for_name(name) - - @memoize_default() - def follow(self, is_goto=False): - module = self._evaluator.wrap(self._import.get_parent_until()) - import_path = self._import.path_for_name(self._name) - from_import_name = None - try: - from_names = self._import.get_from_names() - except AttributeError: - # Is an import_name - pass - else: - if len(from_names) + 1 == len(import_path): - # We have to fetch the from_names part first and then check - # if from_names exists in the modules. - from_import_name = import_path[-1] - import_path = from_names - - importer = Importer(self._evaluator, tuple(import_path), - module, self._import.level) - - types = importer.follow() - - #if self._import.is_nested() and not self.nested_resolve: - # scopes = [NestedImportModule(module, self._import)] - - if from_import_name is not None: - types = set(chain.from_iterable( - self._evaluator.find_types(t, unicode(from_import_name), - is_goto=is_goto) - for t in types)) - - if not types: - path = import_path + [from_import_name] - importer = Importer(self._evaluator, tuple(path), - module, self._import.level) - types = importer.follow() - # goto only accepts `Name` - if is_goto: - types = set(s.name for s in types) - else: - # goto only accepts `Name` - if is_goto: - types = set(s.name for s in types) - - debug.dbg('after import: %s', types) - return types - - -class NestedImportModule(tree.Module): - """ - TODO while there's no use case for nested import module right now, we might - be able to use them for static analysis checks later on. - """ - def __init__(self, module, nested_import): - self._module = module - self._nested_import = nested_import - - def _get_nested_import_name(self): - """ - Generates an Import statement, that can be used to fake nested imports. - """ - i = self._nested_import - # This is not an existing Import statement. Therefore, set position to - # 0 (0 is not a valid line number). - zero = (0, 0) - names = [unicode(name) for name in i.namespace_names[1:]] - name = helpers.FakeName(names, self._nested_import) - new = tree.Import(i._sub_module, zero, zero, name) - new.parent = self._module - debug.dbg('Generated a nested import: %s', new) - return helpers.FakeName(str(i.namespace_names[1]), new) - - def __getattr__(self, name): - return getattr(self._module, name) - - def __repr__(self): - return "<%s: %s of %s>" % (self.__class__.__name__, self._module, - self._nested_import) - - -def _add_error(evaluator, name, message=None): - if hasattr(name, 'parent'): - # Should be a name, not a string! - analysis.add(evaluator, 'import-error', name, message) - - -def get_init_path(directory_path): - """ - The __init__ file can be searched in a directory. If found return it, else - None. - """ - for suffix, _, _ in imp.get_suffixes(): - path = os.path.join(directory_path, '__init__' + suffix) - if os.path.exists(path): - return path - return None - - -class Importer(object): - def __init__(self, evaluator, import_path, module, level=0): - """ - An implementation similar to ``__import__``. Use `follow` - to actually follow the imports. - - *level* specifies whether to use absolute or relative imports. 0 (the - default) means only perform absolute imports. Positive values for level - indicate the number of parent directories to search relative to the - directory of the module calling ``__import__()`` (see PEP 328 for the - details). - - :param import_path: List of namespaces (strings or Names). - """ - debug.speed('import %s' % (import_path,)) - self._evaluator = evaluator - self.level = level - self.module = module - try: - self.file_path = module.py__file__() - except AttributeError: - # Can be None for certain compiled modules like 'builtins'. - self.file_path = None - - if level: - base = module.py__package__().split('.') - if base == ['']: - base = [] - if level > len(base): - path = module.py__file__() - if path is not None: - import_path = list(import_path) - for i in range(level): - path = os.path.dirname(path) - dir_name = os.path.basename(path) - # This is not the proper way to do relative imports. However, since - # Jedi cannot be sure about the entry point, we just calculate an - # absolute path here. - if dir_name: - import_path.insert(0, dir_name) - else: - _add_error(self._evaluator, import_path[-1]) - import_path = [] - # TODO add import error. - debug.warning('Attempted relative import beyond top-level package.') - else: - # Here we basically rewrite the level to 0. - import_path = tuple(base) + tuple(import_path) - self.import_path = import_path - - @property - def str_import_path(self): - """Returns the import path as pure strings instead of `Name`.""" - return tuple(str(name) for name in self.import_path) - - @memoize_default() - def sys_path_with_modifications(self): - in_path = [] - sys_path_mod = list(sys_path.sys_path_with_modifications(self._evaluator, self.module)) - if self.file_path is not None: - # If you edit e.g. gunicorn, there will be imports like this: - # `from gunicorn import something`. But gunicorn is not in the - # sys.path. Therefore look if gunicorn is a parent directory, #56. - if self.import_path: # TODO is this check really needed? - for path in sys_path.traverse_parents(self.file_path): - if os.path.basename(path) == self.str_import_path[0]: - in_path.append(os.path.dirname(path)) - - # Since we know nothing about the call location of the sys.path, - # it's a possibility that the current directory is the origin of - # the Python execution. - sys_path_mod.insert(0, os.path.dirname(self.file_path)) - - return in_path + sys_path_mod - - @memoize_default(NO_DEFAULT) - def follow(self): - if not self.import_path: - return set() - return self._do_import(self.import_path, self.sys_path_with_modifications()) - - def _do_import(self, import_path, sys_path): - """ - This method is very similar to importlib's `_gcd_import`. - """ - import_parts = [str(i) for i in import_path] - - # Handle "magic" Flask extension imports: - # ``flask.ext.foo`` is really ``flask_foo`` or ``flaskext.foo``. - if len(import_path) > 2 and import_parts[:2] == ['flask', 'ext']: - # New style. - ipath = ('flask_' + str(import_parts[2]),) + import_path[3:] - modules = self._do_import(ipath, sys_path) - if modules: - return modules - else: - # Old style - return self._do_import(('flaskext',) + import_path[2:], sys_path) - - module_name = '.'.join(import_parts) - try: - return set([self._evaluator.modules[module_name]]) - except KeyError: - pass - - if len(import_path) > 1: - # This is a recursive way of importing that works great with - # the module cache. - bases = self._do_import(import_path[:-1], sys_path) - if not bases: - return set() - # We can take the first element, because only the os special - # case yields multiple modules, which is not important for - # further imports. - parent_module = list(bases)[0] - - # This is a huge exception, we follow a nested import - # ``os.path``, because it's a very important one in Python - # that is being achieved by messing with ``sys.modules`` in - # ``os``. - if [str(i) for i in import_path] == ['os', 'path']: - return self._evaluator.find_types(parent_module, 'path') - - try: - paths = parent_module.py__path__() - except AttributeError: - # The module is not a package. - _add_error(self._evaluator, import_path[-1]) - return set() - else: - debug.dbg('search_module %s in paths %s', module_name, paths) - for path in paths: - # At the moment we are only using one path. So this is - # not important to be correct. - try: - module_file, module_path, is_pkg = \ - find_module(import_parts[-1], [path]) - break - except ImportError: - module_path = None - if module_path is None: - _add_error(self._evaluator, import_path[-1]) - return set() - else: - parent_module = None - try: - debug.dbg('search_module %s in %s', import_parts[-1], self.file_path) - # Override the sys.path. It works only good that way. - # Injecting the path directly into `find_module` did not work. - sys.path, temp = sys_path, sys.path - try: - module_file, module_path, is_pkg = \ - find_module(import_parts[-1]) - finally: - sys.path = temp - except ImportError: - # The module is not a package. - _add_error(self._evaluator, import_path[-1]) - return set() - - source = None - if is_pkg: - # In this case, we don't have a file yet. Search for the - # __init__ file. - if module_path.endswith(('.zip', '.egg')): - source = module_file.loader.get_source(module_name) - else: - module_path = get_init_path(module_path) - elif module_file: - source = module_file.read() - module_file.close() - - if module_file is None and not module_path.endswith(('.py', '.zip', '.egg')): - module = compiled.load_module(self._evaluator, module_path) - else: - module = _load_module(self._evaluator, module_path, source, sys_path, parent_module) - - if module is None: - # The file might raise an ImportError e.g. and therefore not be - # importable. - return set() - - self._evaluator.modules[module_name] = module - return set([module]) - - def _generate_name(self, name): - # Create a pseudo import to be able to follow them. - name = helpers.FakeName(name) - imp = helpers.FakeImport(name, parent=self.module) - name.parent = imp - return name - - def _get_module_names(self, search_path=None): - """ - Get the names of all modules in the search_path. This means file names - and not names defined in the files. - """ - - names = [] - # add builtin module names - if search_path is None: - names += [self._generate_name(name) for name in sys.builtin_module_names] - - if search_path is None: - search_path = self.sys_path_with_modifications() - for module_loader, name, is_pkg in pkgutil.iter_modules(search_path): - names.append(self._generate_name(name)) - return names - - def completion_names(self, evaluator, only_modules=False): - """ - :param only_modules: Indicates wheter it's possible to import a - definition that is not defined in a module. - """ - from jedi.evaluate import finder - names = [] - if self.import_path: - # flask - if self.str_import_path == ('flask', 'ext'): - # List Flask extensions like ``flask_foo`` - for mod in self._get_module_names(): - modname = str(mod) - if modname.startswith('flask_'): - extname = modname[len('flask_'):] - names.append(self._generate_name(extname)) - # Now the old style: ``flaskext.foo`` - for dir in self.sys_path_with_modifications(): - flaskext = os.path.join(dir, 'flaskext') - if os.path.isdir(flaskext): - names += self._get_module_names([flaskext]) - - for scope in self.follow(): - # Non-modules are not completable. - if not scope.type == 'file_input': # not a module - continue - - # namespace packages - if isinstance(scope, tree.Module) and scope.path.endswith('__init__.py'): - paths = scope.py__path__() - names += self._get_module_names(paths) - - if only_modules: - # In the case of an import like `from x.` we don't need to - # add all the variables. - if ('os',) == self.str_import_path and not self.level: - # os.path is a hardcoded exception, because it's a - # ``sys.modules`` modification. - names.append(self._generate_name('path')) - - continue - - for names_dict in scope.names_dicts(search_global=False): - _names = list(chain.from_iterable(names_dict.values())) - if not _names: - continue - _names = finder.filter_definition_names(_names, scope) - names += _names - else: - # Empty import path=completion after import - if not self.level: - names += self._get_module_names() - - if self.file_path is not None: - path = os.path.abspath(self.file_path) - for i in range(self.level - 1): - path = os.path.dirname(path) - names += self._get_module_names([path]) - - return names - - -def _load_module(evaluator, path=None, source=None, sys_path=None, parent_module=None): - def load(source): - dotted_path = path and compiled.dotted_from_fs_path(path, sys_path) - if path is not None and path.endswith(('.py', '.zip', '.egg')) \ - and dotted_path not in settings.auto_import_modules: - if source is None: - with open(path, 'rb') as f: - source = f.read() - else: - return compiled.load_module(evaluator, path) - p = path - p = fast.FastParser(evaluator.grammar, common.source_to_unicode(source), p) - save_parser(path, p) - from jedi.evaluate.representation import ModuleWrapper - return ModuleWrapper(evaluator, p.module, parent_module) - - if sys_path is None: - sys_path = evaluator.sys_path - - cached = load_parser(path) - module = load(source) if cached is None else cached.module - module = evaluator.wrap(module) - return module - - -def add_module(evaluator, module_name, module): - if '.' not in module_name: - # We cannot add paths with dots, because that would collide with - # the sepatator dots for nested packages. Therefore we return - # `__main__` in ModuleWrapper.py__name__(), which is similar to - # Python behavior. - evaluator.modules[module_name] = module - - -def get_modules_containing_name(evaluator, mods, name): - """ - Search a name in the directories of modules. - """ - def check_python_file(path): - try: - return parser_cache[path].parser.module - except KeyError: - try: - return check_fs(path) - except IOError: - return None - - def check_fs(path): - with open(path, 'rb') as f: - source = source_to_unicode(f.read()) - if name in source: - module_name = os.path.basename(path)[:-3] # Remove `.py`. - module = _load_module(evaluator, path, source) - add_module(evaluator, module_name, module) - return module - - # skip non python modules - mods = set(m for m in mods if not isinstance(m, compiled.CompiledObject)) - mod_paths = set() - for m in mods: - mod_paths.add(m.path) - yield m - - if settings.dynamic_params_for_other_modules: - paths = set(settings.additional_dynamic_modules) - for p in mod_paths: - if p is not None: - # We need abspath, because the seetings paths might not already - # have been converted to absolute paths. - d = os.path.dirname(os.path.abspath(p)) - for entry in os.listdir(d): - if entry not in mod_paths: - if entry.endswith('.py'): - paths.add(d + os.path.sep + entry) - - for p in sorted(paths): - # make testing easier, sort it - same results on every interpreter - c = check_python_file(p) - if c is not None and c not in mods and not isinstance(c, compiled.CompiledObject): - yield c diff --git a/pythonFiles/preview/jedi/evaluate/iterable.py b/pythonFiles/preview/jedi/evaluate/iterable.py deleted file mode 100644 index 79a1a0e26d22..000000000000 --- a/pythonFiles/preview/jedi/evaluate/iterable.py +++ /dev/null @@ -1,863 +0,0 @@ -""" -Contains all classes and functions to deal with lists, dicts, generators and -iterators in general. - -Array modifications -******************* - -If the content of an array (``set``/``list``) is requested somewhere, the -current module will be checked for appearances of ``arr.append``, -``arr.insert``, etc. If the ``arr`` name points to an actual array, the -content will be added - -This can be really cpu intensive, as you can imagine. Because |jedi| has to -follow **every** ``append`` and check wheter it's the right array. However this -works pretty good, because in *slow* cases, the recursion detector and other -settings will stop this process. - -It is important to note that: - -1. Array modfications work only in the current module. -2. Jedi only checks Array additions; ``list.pop``, etc are ignored. -""" -from jedi.common import unite, safe_property -from jedi import debug -from jedi import settings -from jedi._compatibility import use_metaclass, unicode, zip_longest -from jedi.parser import tree -from jedi.evaluate import compiled -from jedi.evaluate import helpers -from jedi.evaluate.cache import CachedMetaClass, memoize_default -from jedi.evaluate import analysis -from jedi.evaluate import pep0484 -from jedi import common - - -class IterableWrapper(tree.Base): - def is_class(self): - return False - - @memoize_default() - def _get_names_dict(self, names_dict): - builtin_methods = {} - for cls in reversed(type(self).mro()): - try: - builtin_methods.update(cls.builtin_methods) - except AttributeError: - pass - - if not builtin_methods: - return names_dict - - dct = {} - for names in names_dict.values(): - for name in names: - name_str = name.value - try: - method = builtin_methods[name_str, self.type] - except KeyError: - dct[name_str] = [name] - else: - parent = BuiltinMethod(self, method, name.parent) - dct[name_str] = [helpers.FakeName(name_str, parent, is_definition=True)] - return dct - - -class BuiltinMethod(IterableWrapper): - """``Generator.__next__`` ``dict.values`` methods and so on.""" - def __init__(self, builtin, method, builtin_func): - self._builtin = builtin - self._method = method - self._builtin_func = builtin_func - - def py__call__(self, params): - return self._method(self._builtin) - - def __getattr__(self, name): - return getattr(self._builtin_func, name) - - -def has_builtin_methods(cls): - cls.builtin_methods = {} - for func in cls.__dict__.values(): - try: - cls.builtin_methods.update(func.registered_builtin_methods) - except AttributeError: - pass - return cls - - -def register_builtin_method(method_name, type=None): - def wrapper(func): - dct = func.__dict__.setdefault('registered_builtin_methods', {}) - dct[method_name, type] = func - return func - return wrapper - - -@has_builtin_methods -class GeneratorMixin(object): - type = None - - @register_builtin_method('send') - @register_builtin_method('next') - @register_builtin_method('__next__') - def py__next__(self): - # TODO add TypeError if params are given. - return unite(self.py__iter__()) - - @memoize_default() - def names_dicts(self, search_global=False): # is always False - gen_obj = compiled.get_special_object(self._evaluator, 'GENERATOR_OBJECT') - yield self._get_names_dict(gen_obj.names_dict) - - def py__bool__(self): - return True - - def py__class__(self): - gen_obj = compiled.get_special_object(self._evaluator, 'GENERATOR_OBJECT') - return gen_obj.py__class__() - - -class Generator(use_metaclass(CachedMetaClass, IterableWrapper, GeneratorMixin)): - """Handling of `yield` functions.""" - - def __init__(self, evaluator, func, var_args): - super(Generator, self).__init__() - self._evaluator = evaluator - self.func = func - self.var_args = var_args - - def py__iter__(self): - from jedi.evaluate.representation import FunctionExecution - f = FunctionExecution(self._evaluator, self.func, self.var_args) - return f.get_yield_types() - - def __getattr__(self, name): - if name not in ['start_pos', 'end_pos', 'parent', 'get_imports', - 'doc', 'docstr', 'get_parent_until', - 'get_code', 'subscopes']: - raise AttributeError("Accessing %s of %s is not allowed." - % (self, name)) - return getattr(self.func, name) - - def __repr__(self): - return "<%s of %s>" % (type(self).__name__, self.func) - - -class Comprehension(IterableWrapper): - @staticmethod - def from_atom(evaluator, atom): - bracket = atom.children[0] - if bracket == '{': - if atom.children[1].children[1] == ':': - cls = DictComprehension - else: - cls = SetComprehension - elif bracket == '(': - cls = GeneratorComprehension - elif bracket == '[': - cls = ListComprehension - return cls(evaluator, atom) - - def __init__(self, evaluator, atom): - self._evaluator = evaluator - self._atom = atom - - def _get_comprehension(self): - # The atom contains a testlist_comp - return self._atom.children[1] - - def _get_comp_for(self): - # The atom contains a testlist_comp - return self._get_comprehension().children[1] - - @memoize_default() - def _eval_node(self, index=0): - """ - The first part `x + 1` of the list comprehension: - - [x + 1 for x in foo] - """ - comp_for = self._get_comp_for() - # For nested comprehensions we need to search the last one. - from jedi.evaluate.representation import InstanceElement - node = self._get_comprehension().children[index] - if isinstance(node, InstanceElement): - # This seems to be a strange case that I haven't found a way to - # write tests against. However since it's my new goal to get rid of - # InstanceElement anyway, I don't care. - node = node.var - last_comp = list(comp_for.get_comp_fors())[-1] - return helpers.deep_ast_copy(node, parent=last_comp) - - def _nested(self, comp_fors): - evaluator = self._evaluator - comp_for = comp_fors[0] - input_node = comp_for.children[3] - input_types = evaluator.eval_element(input_node) - - iterated = py__iter__(evaluator, input_types, input_node) - exprlist = comp_for.children[1] - for i, types in enumerate(iterated): - evaluator.predefined_if_name_dict_dict[comp_for] = \ - unpack_tuple_to_dict(evaluator, types, exprlist) - try: - for result in self._nested(comp_fors[1:]): - yield result - except IndexError: - iterated = evaluator.eval_element(self._eval_node()) - if self.type == 'dict': - yield iterated, evaluator.eval_element(self._eval_node(2)) - else: - yield iterated - finally: - del evaluator.predefined_if_name_dict_dict[comp_for] - - @memoize_default(default=[]) - @common.to_list - def _iterate(self): - comp_fors = tuple(self._get_comp_for().get_comp_fors()) - for result in self._nested(comp_fors): - yield result - - def py__iter__(self): - return self._iterate() - - def __repr__(self): - return "<%s of %s>" % (type(self).__name__, self._atom) - - -@has_builtin_methods -class ArrayMixin(object): - @memoize_default() - def names_dicts(self, search_global=False): # Always False. - # `array.type` is a string with the type, e.g. 'list'. - scope = compiled.builtin_from_name(self._evaluator, self.type) - # builtins only have one class -> [0] - scopes = self._evaluator.execute_evaluated(scope, self) - names_dicts = list(scopes)[0].names_dicts(search_global) - #yield names_dicts[0] - yield self._get_names_dict(names_dicts[1]) - - def py__bool__(self): - return None # We don't know the length, because of appends. - - def py__class__(self): - return compiled.builtin_from_name(self._evaluator, self.type) - - @safe_property - def parent(self): - return self._evaluator.BUILTINS - - @property - def name(self): - return FakeSequence(self._evaluator, [], self.type).name - - @memoize_default() - def dict_values(self): - return unite(self._evaluator.eval_element(v) for k, v in self._items()) - - @register_builtin_method('values', type='dict') - def _imitate_values(self): - items = self.dict_values() - return create_evaluated_sequence_set(self._evaluator, items, sequence_type='list') - #return set([FakeSequence(self._evaluator, [AlreadyEvaluated(items)], 'tuple')]) - - @register_builtin_method('items', type='dict') - def _imitate_items(self): - items = [set([FakeSequence(self._evaluator, (k, v), 'tuple')]) - for k, v in self._items()] - - return create_evaluated_sequence_set(self._evaluator, *items, sequence_type='list') - - -class ListComprehension(Comprehension, ArrayMixin): - type = 'list' - - def py__getitem__(self, index): - all_types = list(self.py__iter__()) - result = all_types[index] - if isinstance(index, slice): - return create_evaluated_sequence_set( - self._evaluator, - unite(result), - sequence_type='list' - ) - return result - - -class SetComprehension(Comprehension, ArrayMixin): - type = 'set' - - -@has_builtin_methods -class DictComprehension(Comprehension, ArrayMixin): - type = 'dict' - - def _get_comp_for(self): - return self._get_comprehension().children[3] - - def py__iter__(self): - for keys, values in self._iterate(): - yield keys - - def py__getitem__(self, index): - for keys, values in self._iterate(): - for k in keys: - if isinstance(k, compiled.CompiledObject): - if k.obj == index: - return values - return self.dict_values() - - def dict_values(self): - return unite(values for keys, values in self._iterate()) - - @register_builtin_method('items', type='dict') - def _imitate_items(self): - items = set(FakeSequence(self._evaluator, - (AlreadyEvaluated(keys), AlreadyEvaluated(values)), 'tuple') - for keys, values in self._iterate()) - - return create_evaluated_sequence_set(self._evaluator, items, sequence_type='list') - - -class GeneratorComprehension(Comprehension, GeneratorMixin): - pass - - -class Array(IterableWrapper, ArrayMixin): - mapping = {'(': 'tuple', - '[': 'list', - '{': 'dict'} - - def __init__(self, evaluator, atom): - self._evaluator = evaluator - self.atom = atom - self.type = Array.mapping[atom.children[0]] - """The builtin name of the array (list, set, tuple or dict).""" - - c = self.atom.children - array_node = c[1] - if self.type == 'dict' and array_node != '}' \ - and (not hasattr(array_node, 'children') - or ':' not in array_node.children): - self.type = 'set' - - @property - def name(self): - return helpers.FakeName(self.type, parent=self) - - def py__getitem__(self, index): - """Here the index is an int/str. Raises IndexError/KeyError.""" - if self.type == 'dict': - for key, value in self._items(): - for k in self._evaluator.eval_element(key): - if isinstance(k, compiled.CompiledObject) \ - and index == k.obj: - return self._evaluator.eval_element(value) - raise KeyError('No key found in dictionary %s.' % self) - - # Can raise an IndexError - if isinstance(index, slice): - return set([self]) - else: - return self._evaluator.eval_element(self._items()[index]) - - def __getattr__(self, name): - if name not in ['start_pos', 'get_only_subelement', 'parent', - 'get_parent_until', 'items']: - raise AttributeError('Strange access on %s: %s.' % (self, name)) - return getattr(self.atom, name) - - # @memoize_default() - def py__iter__(self): - """ - While values returns the possible values for any array field, this - function returns the value for a certain index. - """ - if self.type == 'dict': - # Get keys. - types = set() - for k, _ in self._items(): - types |= self._evaluator.eval_element(k) - # We don't know which dict index comes first, therefore always - # yield all the types. - for _ in types: - yield types - else: - for value in self._items(): - yield self._evaluator.eval_element(value) - - additions = check_array_additions(self._evaluator, self) - if additions: - yield additions - - def _values(self): - """Returns a list of a list of node.""" - if self.type == 'dict': - return unite(v for k, v in self._items()) - else: - return self._items() - - def _items(self): - c = self.atom.children - array_node = c[1] - if array_node in (']', '}', ')'): - return [] # Direct closing bracket, doesn't contain items. - - if tree.is_node(array_node, 'testlist_comp'): - return array_node.children[::2] - elif tree.is_node(array_node, 'dictorsetmaker'): - kv = [] - iterator = iter(array_node.children) - for key in iterator: - op = next(iterator, None) - if op is None or op == ',': - kv.append(key) # A set. - else: - assert op == ':' # A dict. - kv.append((key, next(iterator))) - next(iterator, None) # Possible comma. - return kv - else: - return [array_node] - - def __repr__(self): - return "<%s of %s>" % (type(self).__name__, self.atom) - - -class _FakeArray(Array): - def __init__(self, evaluator, container, type): - self.type = type - self._evaluator = evaluator - self.atom = container - - -class ImplicitTuple(_FakeArray): - def __init__(self, evaluator, testlist): - super(ImplicitTuple, self).__init__(evaluator, testlist, 'tuple') - self._testlist = testlist - - def _items(self): - return self._testlist.children[::2] - - -class FakeSequence(_FakeArray): - def __init__(self, evaluator, sequence_values, type): - """ - type should be one of "tuple", "list" - """ - super(FakeSequence, self).__init__(evaluator, sequence_values, type) - self._sequence_values = sequence_values - - def _items(self): - return self._sequence_values - - -def create_evaluated_sequence_set(evaluator, *types_order, **kwargs): - """ - ``sequence_type`` is a named argument, that doesn't work in Python2. For backwards - compatibility reasons, we're now using kwargs. - """ - sequence_type = kwargs.pop('sequence_type') - assert not kwargs - - sets = tuple(AlreadyEvaluated(types) for types in types_order) - return set([FakeSequence(evaluator, sets, sequence_type)]) - - -class AlreadyEvaluated(frozenset): - """A simple container to add already evaluated objects to an array.""" - def get_code(self, normalized=False): - # For debugging purposes. - return str(self) - - -class MergedNodes(frozenset): - pass - - -class FakeDict(_FakeArray): - def __init__(self, evaluator, dct): - super(FakeDict, self).__init__(evaluator, dct, 'dict') - self._dct = dct - - def py__iter__(self): - yield set(compiled.create(self._evaluator, key) for key in self._dct) - - def py__getitem__(self, index): - return unite(self._evaluator.eval_element(v) for v in self._dct[index]) - - def _items(self): - for key, values in self._dct.items(): - # TODO this is not proper. The values could be multiple values?! - yield key, values[0] - - -class MergedArray(_FakeArray): - def __init__(self, evaluator, arrays): - super(MergedArray, self).__init__(evaluator, arrays, arrays[-1].type) - self._arrays = arrays - - def py__iter__(self): - for array in self._arrays: - for types in array.py__iter__(): - yield types - - def py__getitem__(self, index): - return unite(self.py__iter__()) - - def _items(self): - for array in self._arrays: - for a in array._items(): - yield a - - def __len__(self): - return sum(len(a) for a in self._arrays) - - -def unpack_tuple_to_dict(evaluator, types, exprlist): - """ - Unpacking tuple assignments in for statements and expr_stmts. - """ - if exprlist.type == 'name': - return {exprlist.value: types} - elif exprlist.type == 'atom' and exprlist.children[0] in '([': - return unpack_tuple_to_dict(evaluator, types, exprlist.children[1]) - elif exprlist.type in ('testlist', 'testlist_comp', 'exprlist', - 'testlist_star_expr'): - dct = {} - parts = iter(exprlist.children[::2]) - n = 0 - for iter_types in py__iter__(evaluator, types, exprlist): - n += 1 - try: - part = next(parts) - except StopIteration: - analysis.add(evaluator, 'value-error-too-many-values', part, - message="ValueError: too many values to unpack (expected %s)" % n) - else: - dct.update(unpack_tuple_to_dict(evaluator, iter_types, part)) - has_parts = next(parts, None) - if types and has_parts is not None: - analysis.add(evaluator, 'value-error-too-few-values', has_parts, - message="ValueError: need more than %s values to unpack" % n) - return dct - elif exprlist.type == 'power' or exprlist.type == 'atom_expr': - # Something like ``arr[x], var = ...``. - # This is something that is not yet supported, would also be difficult - # to write into a dict. - return {} - elif exprlist.type == 'star_expr': # `a, *b, c = x` type unpackings - # Currently we're not supporting them. - return {} - raise NotImplementedError - - -def py__iter__(evaluator, types, node=None): - debug.dbg('py__iter__') - type_iters = [] - for typ in types: - try: - iter_method = typ.py__iter__ - except AttributeError: - if node is not None: - analysis.add(evaluator, 'type-error-not-iterable', node, - message="TypeError: '%s' object is not iterable" % typ) - else: - type_iters.append(iter_method()) - #for result in iter_method(): - #yield result - - for t in zip_longest(*type_iters, fillvalue=set()): - yield unite(t) - - -def py__iter__types(evaluator, types, node=None): - """ - Calls `py__iter__`, but ignores the ordering in the end and just returns - all types that it contains. - """ - return unite(py__iter__(evaluator, types, node)) - - -def py__getitem__(evaluator, types, trailer): - from jedi.evaluate.representation import Class - result = set() - - trailer_op, node, trailer_cl = trailer.children - assert trailer_op == "[" - assert trailer_cl == "]" - - # special case: PEP0484 typing module, see - # https://github.com/davidhalter/jedi/issues/663 - for typ in list(types): - if isinstance(typ, Class): - typing_module_types = \ - pep0484.get_types_for_typing_module(evaluator, typ, node) - if typing_module_types is not None: - types.remove(typ) - result |= typing_module_types - - if not types: - # all consumed by special cases - return result - - for index in create_index_types(evaluator, node): - if isinstance(index, (compiled.CompiledObject, Slice)): - index = index.obj - - if type(index) not in (float, int, str, unicode, slice): - # If the index is not clearly defined, we have to get all the - # possiblities. - for typ in list(types): - if isinstance(typ, Array) and typ.type == 'dict': - types.remove(typ) - result |= typ.dict_values() - return result | py__iter__types(evaluator, types) - - for typ in types: - # The actual getitem call. - try: - getitem = typ.py__getitem__ - except AttributeError: - analysis.add(evaluator, 'type-error-not-subscriptable', trailer_op, - message="TypeError: '%s' object is not subscriptable" % typ) - else: - try: - result |= getitem(index) - except IndexError: - result |= py__iter__types(evaluator, set([typ])) - except KeyError: - # Must be a dict. Lists don't raise KeyErrors. - result |= typ.dict_values() - return result - - -def check_array_additions(evaluator, array): - """ Just a mapper function for the internal _check_array_additions """ - if array.type not in ('list', 'set'): - # TODO also check for dict updates - return set() - - is_list = array.type == 'list' - try: - current_module = array.atom.get_parent_until() - except AttributeError: - # If there's no get_parent_until, it's a FakeSequence or another Fake - # type. Those fake types are used inside Jedi's engine. No values may - # be added to those after their creation. - return set() - return _check_array_additions(evaluator, array, current_module, is_list) - - -@memoize_default(default=set(), evaluator_is_first_arg=True) -@debug.increase_indent -def _check_array_additions(evaluator, compare_array, module, is_list): - """ - Checks if a `Array` has "add" (append, insert, extend) statements: - - >>> a = [""] - >>> a.append(1) - """ - debug.dbg('Dynamic array search for %s' % compare_array, color='MAGENTA') - if not settings.dynamic_array_additions or isinstance(module, compiled.CompiledObject): - debug.dbg('Dynamic array search aborted.', color='MAGENTA') - return set() - - def check_additions(arglist, add_name): - params = list(param.Arguments(evaluator, arglist).unpack()) - result = set() - if add_name in ['insert']: - params = params[1:] - if add_name in ['append', 'add', 'insert']: - for key, nodes in params: - result |= unite(evaluator.eval_element(node) for node in nodes) - elif add_name in ['extend', 'update']: - for key, nodes in params: - for node in nodes: - types = evaluator.eval_element(node) - result |= py__iter__types(evaluator, types, node) - return result - - from jedi.evaluate import representation as er, param - - def get_execution_parent(element): - """ Used to get an Instance/FunctionExecution parent """ - if isinstance(element, Array): - node = element.atom - else: - # Is an Instance with an - # Arguments([AlreadyEvaluated([_ArrayInstance])]) inside - # Yeah... I know... It's complicated ;-) - node = list(element.var_args.argument_node[0])[0].var_args.trailer - if isinstance(node, er.InstanceElement) or node is None: - return node - return node.get_parent_until(er.FunctionExecution) - - temp_param_add, settings.dynamic_params_for_other_modules = \ - settings.dynamic_params_for_other_modules, False - - search_names = ['append', 'extend', 'insert'] if is_list else ['add', 'update'] - comp_arr_parent = get_execution_parent(compare_array) - - added_types = set() - for add_name in search_names: - try: - possible_names = module.used_names[add_name] - except KeyError: - continue - else: - for name in possible_names: - # Check if the original scope is an execution. If it is, one - # can search for the same statement, that is in the module - # dict. Executions are somewhat special in jedi, since they - # literally copy the contents of a function. - if isinstance(comp_arr_parent, er.FunctionExecution): - if comp_arr_parent.start_pos < name.start_pos < comp_arr_parent.end_pos: - name = comp_arr_parent.name_for_position(name.start_pos) - else: - # Don't check definitions that are not defined in the - # same function. This is not "proper" anyway. It also - # improves Jedi's speed for array lookups, since we - # don't have to check the whole source tree anymore. - continue - trailer = name.parent - power = trailer.parent - trailer_pos = power.children.index(trailer) - try: - execution_trailer = power.children[trailer_pos + 1] - except IndexError: - continue - else: - if execution_trailer.type != 'trailer' \ - or execution_trailer.children[0] != '(' \ - or execution_trailer.children[1] == ')': - continue - power = helpers.call_of_leaf(name, cut_own_trailer=True) - # InstanceElements are special, because they don't get copied, - # but have this wrapper around them. - if isinstance(comp_arr_parent, er.InstanceElement): - power = er.get_instance_el(evaluator, comp_arr_parent.instance, power) - - if evaluator.recursion_detector.push_stmt(power): - # Check for recursion. Possible by using 'extend' in - # combination with function calls. - continue - try: - if compare_array in evaluator.eval_element(power): - # The arrays match. Now add the results - added_types |= check_additions(execution_trailer.children[1], add_name) - finally: - evaluator.recursion_detector.pop_stmt() - # reset settings - settings.dynamic_params_for_other_modules = temp_param_add - debug.dbg('Dynamic array result %s' % added_types, color='MAGENTA') - return added_types - - -def check_array_instances(evaluator, instance): - """Used for set() and list() instances.""" - if not settings.dynamic_array_additions: - return instance.var_args - - ai = _ArrayInstance(evaluator, instance) - from jedi.evaluate import param - return param.Arguments(evaluator, [AlreadyEvaluated([ai])]) - - -class _ArrayInstance(IterableWrapper): - """ - Used for the usage of set() and list(). - This is definitely a hack, but a good one :-) - It makes it possible to use set/list conversions. - - In contrast to Array, ListComprehension and all other iterable types, this - is something that is only used inside `evaluate/compiled/fake/builtins.py` - and therefore doesn't need `names_dicts`, `py__bool__` and so on, because - we don't use these operations in `builtins.py`. - """ - def __init__(self, evaluator, instance): - self._evaluator = evaluator - self.instance = instance - self.var_args = instance.var_args - - def py__iter__(self): - try: - _, first_nodes = next(self.var_args.unpack()) - except StopIteration: - types = set() - else: - types = unite(self._evaluator.eval_element(node) for node in first_nodes) - for types in py__iter__(self._evaluator, types, first_nodes[0]): - yield types - - module = self.var_args.get_parent_until() - if module is None: - return - is_list = str(self.instance.name) == 'list' - additions = _check_array_additions(self._evaluator, self.instance, module, is_list) - if additions: - yield additions - - -class Slice(object): - def __init__(self, evaluator, start, stop, step): - self._evaluator = evaluator - # all of them are either a Precedence or None. - self._start = start - self._stop = stop - self._step = step - - @property - def obj(self): - """ - Imitate CompiledObject.obj behavior and return a ``builtin.slice()`` - object. - """ - def get(element): - if element is None: - return None - - result = self._evaluator.eval_element(element) - if len(result) != 1: - # For simplicity, we want slices to be clear defined with just - # one type. Otherwise we will return an empty slice object. - raise IndexError - try: - return list(result)[0].obj - except AttributeError: - return None - - try: - return slice(get(self._start), get(self._stop), get(self._step)) - except IndexError: - return slice(None, None, None) - - -def create_index_types(evaluator, index): - """ - Handles slices in subscript nodes. - """ - if index == ':': - # Like array[:] - return set([Slice(evaluator, None, None, None)]) - elif tree.is_node(index, 'subscript'): # subscript is a slice operation. - # Like array[:3] - result = [] - for el in index.children: - if el == ':': - if not result: - result.append(None) - elif tree.is_node(el, 'sliceop'): - if len(el.children) == 2: - result.append(el.children[1]) - else: - result.append(el) - result += [None] * (3 - len(result)) - - return set([Slice(evaluator, *result)]) - - # No slices - return evaluator.eval_element(index) diff --git a/pythonFiles/preview/jedi/evaluate/param.py b/pythonFiles/preview/jedi/evaluate/param.py deleted file mode 100644 index 997e07992b24..000000000000 --- a/pythonFiles/preview/jedi/evaluate/param.py +++ /dev/null @@ -1,438 +0,0 @@ -from collections import defaultdict -from itertools import chain - -from jedi._compatibility import unicode, zip_longest -from jedi import debug -from jedi import common -from jedi.parser import tree -from jedi.evaluate import iterable -from jedi.evaluate import analysis -from jedi.evaluate import precedence -from jedi.evaluate.helpers import FakeName -from jedi.cache import underscore_memoization - - -def try_iter_content(types, depth=0): - """Helper method for static analysis.""" - if depth > 10: - # It's possible that a loop has references on itself (especially with - # CompiledObject). Therefore don't loop infinitely. - return - - for typ in types: - try: - f = typ.py__iter__ - except AttributeError: - pass - else: - for iter_types in f(): - try_iter_content(iter_types, depth + 1) - - -class Arguments(tree.Base): - def __init__(self, evaluator, argument_node, trailer=None): - """ - The argument_node is either a parser node or a list of evaluated - objects. Those evaluated objects may be lists of evaluated objects - themselves (one list for the first argument, one for the second, etc). - - :param argument_node: May be an argument_node or a list of nodes. - """ - self.argument_node = argument_node - self._evaluator = evaluator - self.trailer = trailer # Can be None, e.g. in a class definition. - - def _split(self): - if isinstance(self.argument_node, (tuple, list)): - for el in self.argument_node: - yield 0, el - else: - if not (tree.is_node(self.argument_node, 'arglist') or ( - # in python 3.5 **arg is an argument, not arglist - (tree.is_node(self.argument_node, 'argument') and - self.argument_node.children[0] in ('*', '**')))): - yield 0, self.argument_node - return - - iterator = iter(self.argument_node.children) - for child in iterator: - if child == ',': - continue - elif child in ('*', '**'): - yield len(child.value), next(iterator) - elif tree.is_node(child, 'argument') and \ - child.children[0] in ('*', '**'): - assert len(child.children) == 2 - yield len(child.children[0].value), child.children[1] - else: - yield 0, child - - def get_parent_until(self, *args, **kwargs): - if self.trailer is None: - try: - element = self.argument_node[0] - from jedi.evaluate.iterable import AlreadyEvaluated - if isinstance(element, AlreadyEvaluated): - element = list(self._evaluator.eval_element(element))[0] - except IndexError: - return None - else: - return element.get_parent_until(*args, **kwargs) - else: - return self.trailer.get_parent_until(*args, **kwargs) - - def as_tuple(self): - for stars, argument in self._split(): - if tree.is_node(argument, 'argument'): - argument, default = argument.children[::2] - else: - default = None - yield argument, default, stars - - def unpack(self, func=None): - named_args = [] - for stars, el in self._split(): - if stars == 1: - arrays = self._evaluator.eval_element(el) - iterators = [_iterate_star_args(self._evaluator, a, el, func) - for a in arrays] - iterators = list(iterators) - for values in list(zip_longest(*iterators)): - yield None, [v for v in values if v is not None] - elif stars == 2: - arrays = self._evaluator.eval_element(el) - dicts = [_star_star_dict(self._evaluator, a, el, func) - for a in arrays] - for dct in dicts: - for key, values in dct.items(): - yield key, values - else: - if tree.is_node(el, 'argument'): - c = el.children - if len(c) == 3: # Keyword argument. - named_args.append((c[0].value, (c[2],))) - else: # Generator comprehension. - # Include the brackets with the parent. - comp = iterable.GeneratorComprehension( - self._evaluator, self.argument_node.parent) - yield None, (iterable.AlreadyEvaluated([comp]),) - elif isinstance(el, (list, tuple)): - yield None, el - else: - yield None, (el,) - - # Reordering var_args is necessary, because star args sometimes appear - # after named argument, but in the actual order it's prepended. - for key_arg in named_args: - yield key_arg - - def _reorder_var_args(var_args): - named_index = None - new_args = [] - for i, stmt in enumerate(var_args): - if isinstance(stmt, tree.ExprStmt): - if named_index is None and stmt.assignment_details: - named_index = i - - if named_index is not None: - expression_list = stmt.expression_list() - if expression_list and expression_list[0] == '*': - new_args.insert(named_index, stmt) - named_index += 1 - continue - - new_args.append(stmt) - return new_args - - def eval_argument_clinic(self, arguments): - """Uses a list with argument clinic information (see PEP 436).""" - iterator = self.unpack() - for i, (name, optional, allow_kwargs) in enumerate(arguments): - key, va_values = next(iterator, (None, [])) - if key is not None: - raise NotImplementedError - if not va_values and not optional: - debug.warning('TypeError: %s expected at least %s arguments, got %s', - name, len(arguments), i) - raise ValueError - values = set(chain.from_iterable(self._evaluator.eval_element(el) - for el in va_values)) - if not values and not optional: - # For the stdlib we always want values. If we don't get them, - # that's ok, maybe something is too hard to resolve, however, - # we will not proceed with the evaluation of that function. - debug.warning('argument_clinic "%s" not resolvable.', name) - raise ValueError - yield values - - def scope(self): - # Returns the scope in which the arguments are used. - return (self.trailer or self.argument_node).get_parent_until(tree.IsScope) - - def eval_args(self): - # TODO this method doesn't work with named args and a lot of other - # things. Use unpack. - return [self._evaluator.eval_element(el) for stars, el in self._split()] - - def __repr__(self): - return '<%s: %s>' % (type(self).__name__, self.argument_node) - - def get_calling_var_args(self): - if tree.is_node(self.argument_node, 'arglist', 'argument') \ - or self.argument_node == () and self.trailer is not None: - return _get_calling_var_args(self._evaluator, self) - else: - return None - - def eval_all(self, func=None): - """ - Evaluates all arguments as a support for static analysis - (normally Jedi). - """ - for key, element_values in self.unpack(): - for element in element_values: - types = self._evaluator.eval_element(element) - try_iter_content(types) - - -class ExecutedParam(tree.Param): - """Fake a param and give it values.""" - def __init__(self, original_param, var_args, values): - self._original_param = original_param - self.var_args = var_args - self._values = values - - def eval(self, evaluator): - types = set() - for v in self._values: - types |= evaluator.eval_element(v) - return types - - @property - def position_nr(self): - # Need to use the original logic here, because it uses the parent. - return self._original_param.position_nr - - @property - @underscore_memoization - def name(self): - return FakeName(str(self._original_param.name), self, self.start_pos) - - def __getattr__(self, name): - return getattr(self._original_param, name) - - -def _get_calling_var_args(evaluator, var_args): - old_var_args = None - while var_args != old_var_args: - old_var_args = var_args - for name, default, stars in reversed(list(var_args.as_tuple())): - if not stars or not isinstance(name, tree.Name): - continue - - names = evaluator.goto(name) - if len(names) != 1: - break - param = names[0].get_definition() - if not isinstance(param, ExecutedParam): - if isinstance(param, tree.Param): - # There is no calling var_args in this case - there's just - # a param without any input. - return None - break - # We never want var_args to be a tuple. This should be enough for - # now, we can change it later, if we need to. - if isinstance(param.var_args, Arguments): - var_args = param.var_args - return var_args.argument_node or var_args.trailer - - -def get_params(evaluator, func, var_args): - param_names = [] - param_dict = {} - for param in func.params: - param_dict[str(param.name)] = param - unpacked_va = list(var_args.unpack(func)) - from jedi.evaluate.representation import InstanceElement - if isinstance(func, InstanceElement): - # Include self at this place. - unpacked_va.insert(0, (None, [iterable.AlreadyEvaluated([func.instance])])) - var_arg_iterator = common.PushBackIterator(iter(unpacked_va)) - - non_matching_keys = defaultdict(lambda: []) - keys_used = {} - keys_only = False - had_multiple_value_error = False - for param in func.params: - # The value and key can both be null. There, the defaults apply. - # args / kwargs will just be empty arrays / dicts, respectively. - # Wrong value count is just ignored. If you try to test cases that are - # not allowed in Python, Jedi will maybe not show any completions. - default = [] if param.default is None else [param.default] - key, va_values = next(var_arg_iterator, (None, default)) - while key is not None: - keys_only = True - k = unicode(key) - try: - key_param = param_dict[unicode(key)] - except KeyError: - non_matching_keys[key] = va_values - else: - param_names.append(ExecutedParam(key_param, var_args, va_values).name) - - if k in keys_used: - had_multiple_value_error = True - m = ("TypeError: %s() got multiple values for keyword argument '%s'." - % (func.name, k)) - calling_va = _get_calling_var_args(evaluator, var_args) - if calling_va is not None: - analysis.add(evaluator, 'type-error-multiple-values', - calling_va, message=m) - else: - try: - keys_used[k] = param_names[-1] - except IndexError: - # TODO this is wrong stupid and whatever. - pass - key, va_values = next(var_arg_iterator, (None, ())) - - values = [] - if param.stars == 1: - # *args param - lst_values = [iterable.MergedNodes(va_values)] if va_values else [] - for key, va_values in var_arg_iterator: - # Iterate until a key argument is found. - if key: - var_arg_iterator.push_back((key, va_values)) - break - if va_values: - lst_values.append(iterable.MergedNodes(va_values)) - seq = iterable.FakeSequence(evaluator, lst_values, 'tuple') - values = [iterable.AlreadyEvaluated([seq])] - elif param.stars == 2: - # **kwargs param - dct = iterable.FakeDict(evaluator, dict(non_matching_keys)) - values = [iterable.AlreadyEvaluated([dct])] - non_matching_keys = {} - else: - # normal param - if va_values: - values = va_values - else: - # No value: Return an empty container - values = [] - if not keys_only: - calling_va = var_args.get_calling_var_args() - if calling_va is not None: - m = _error_argument_count(func, len(unpacked_va)) - analysis.add(evaluator, 'type-error-too-few-arguments', - calling_va, message=m) - - # Now add to result if it's not one of the previously covered cases. - if (not keys_only or param.stars == 2): - param_names.append(ExecutedParam(param, var_args, values).name) - keys_used[unicode(param.name)] = param_names[-1] - - if keys_only: - # All arguments should be handed over to the next function. It's not - # about the values inside, it's about the names. Jedi needs to now that - # there's nothing to find for certain names. - for k in set(param_dict) - set(keys_used): - param = param_dict[k] - values = [] if param.default is None else [param.default] - param_names.append(ExecutedParam(param, var_args, values).name) - - if not (non_matching_keys or had_multiple_value_error - or param.stars or param.default): - # add a warning only if there's not another one. - calling_va = _get_calling_var_args(evaluator, var_args) - if calling_va is not None: - m = _error_argument_count(func, len(unpacked_va)) - analysis.add(evaluator, 'type-error-too-few-arguments', - calling_va, message=m) - - for key, va_values in non_matching_keys.items(): - m = "TypeError: %s() got an unexpected keyword argument '%s'." \ - % (func.name, key) - for value in va_values: - analysis.add(evaluator, 'type-error-keyword-argument', value.parent, message=m) - - remaining_params = list(var_arg_iterator) - if remaining_params: - m = _error_argument_count(func, len(unpacked_va)) - # Just report an error for the first param that is not needed (like - # cPython). - first_key, first_values = remaining_params[0] - for v in first_values: - if first_key is not None: - # Is a keyword argument, return the whole thing instead of just - # the value node. - v = v.parent - try: - non_kw_param = keys_used[first_key] - except KeyError: - pass - else: - origin_args = non_kw_param.parent.var_args.argument_node - # TODO calculate the var_args tree and check if it's in - # the tree (if not continue). - # print('\t\tnonkw', non_kw_param.parent.var_args.argument_node, ) - if origin_args not in [f.parent.parent for f in first_values]: - continue - analysis.add(evaluator, 'type-error-too-many-arguments', - v, message=m) - return param_names - - -def _iterate_star_args(evaluator, array, input_node, func=None): - from jedi.evaluate.representation import Instance - if isinstance(array, iterable.Array): - # TODO ._items is not the call we want here. Replace in the future. - for node in array._items(): - yield node - elif isinstance(array, iterable.Generator): - for types in array.py__iter__(): - yield iterable.AlreadyEvaluated(types) - elif isinstance(array, Instance) and array.name.get_code() == 'tuple': - debug.warning('Ignored a tuple *args input %s' % array) - else: - if func is not None: - m = "TypeError: %s() argument after * must be a sequence, not %s" \ - % (func.name.value, array) - analysis.add(evaluator, 'type-error-star', input_node, message=m) - - -def _star_star_dict(evaluator, array, input_node, func): - dct = defaultdict(lambda: []) - from jedi.evaluate.representation import Instance - if isinstance(array, Instance) and array.name.get_code() == 'dict': - # For now ignore this case. In the future add proper iterators and just - # make one call without crazy isinstance checks. - return {} - - if isinstance(array, iterable.FakeDict): - return array._dct - elif isinstance(array, iterable.Array) and array.type == 'dict': - # TODO bad call to non-public API - for key_node, value in array._items(): - for key in evaluator.eval_element(key_node): - if precedence.is_string(key): - dct[key.obj].append(value) - - else: - if func is not None: - m = "TypeError: %s argument after ** must be a mapping, not %s" \ - % (func.name.value, array) - analysis.add(evaluator, 'type-error-star-star', input_node, message=m) - return dict(dct) - - -def _error_argument_count(func, actual_count): - default_arguments = sum(1 for p in func.params if p.default or p.stars) - - if default_arguments == 0: - before = 'exactly ' - else: - before = 'from %s to ' % (len(func.params) - default_arguments) - return ('TypeError: %s() takes %s%s arguments (%s given).' - % (func.name, before, len(func.params), actual_count)) diff --git a/pythonFiles/preview/jedi/evaluate/pep0484.py b/pythonFiles/preview/jedi/evaluate/pep0484.py deleted file mode 100644 index 2387fe64a1e1..000000000000 --- a/pythonFiles/preview/jedi/evaluate/pep0484.py +++ /dev/null @@ -1,195 +0,0 @@ -""" -PEP 0484 ( https://www.python.org/dev/peps/pep-0484/ ) describes type hints -through function annotations. There is a strong suggestion in this document -that only the type of type hinting defined in PEP0484 should be allowed -as annotations in future python versions. - -The (initial / probably incomplete) implementation todo list for pep-0484: -v Function parameter annotations with builtin/custom type classes -v Function returntype annotations with builtin/custom type classes -v Function parameter annotations with strings (forward reference) -v Function return type annotations with strings (forward reference) -v Local variable type hints -v Assigned types: `Url = str\ndef get(url:Url) -> str:` -v Type hints in `with` statements -x Stub files support -x support `@no_type_check` and `@no_type_check_decorator` -x support for typing.cast() operator -x support for type hint comments for functions, `# type: (int, str) -> int`. - See comment from Guido https://github.com/davidhalter/jedi/issues/662 -""" - -import itertools - -import os -from jedi.parser import \ - Parser, load_grammar, ParseError, ParserWithRecovery, tree -from jedi.evaluate.cache import memoize_default -from jedi.common import unite -from jedi.evaluate import compiled -from jedi import debug -from jedi import _compatibility -import re - - -def _evaluate_for_annotation(evaluator, annotation, index=None): - """ - Evaluates a string-node, looking for an annotation - If index is not None, the annotation is expected to be a tuple - and we're interested in that index - """ - if annotation is not None: - definitions = evaluator.eval_element( - _fix_forward_reference(evaluator, annotation)) - if index is not None: - definitions = list(itertools.chain.from_iterable( - definition.py__getitem__(index) for definition in definitions - if definition.type == 'tuple' and - len(list(definition.py__iter__())) >= index)) - return list(itertools.chain.from_iterable( - evaluator.execute(d) for d in definitions)) - else: - return [] - - -def _fix_forward_reference(evaluator, node): - evaled_nodes = evaluator.eval_element(node) - if len(evaled_nodes) != 1: - debug.warning("Eval'ed typing index %s should lead to 1 object, " - " not %s" % (node, evaled_nodes)) - return node - evaled_node = list(evaled_nodes)[0] - if isinstance(evaled_node, compiled.CompiledObject) and \ - isinstance(evaled_node.obj, str): - try: - p = Parser(load_grammar(), _compatibility.unicode(evaled_node.obj), - start_symbol='eval_input') - newnode = p.get_parsed_node() - except ParseError: - debug.warning('Annotation not parsed: %s' % evaled_node.obj) - return node - else: - module = node.get_parent_until() - p.position_modifier.line = module.end_pos[0] - newnode.parent = module - return newnode - else: - return node - - -@memoize_default(None, evaluator_is_first_arg=True) -def follow_param(evaluator, param): - annotation = param.annotation() - return _evaluate_for_annotation(evaluator, annotation) - - -@memoize_default(None, evaluator_is_first_arg=True) -def find_return_types(evaluator, func): - annotation = func.py__annotations__().get("return", None) - return _evaluate_for_annotation(evaluator, annotation) - - -_typing_module = None - - -def _get_typing_replacement_module(): - """ - The idea is to return our jedi replacement for the PEP-0484 typing module - as discussed at https://github.com/davidhalter/jedi/issues/663 - """ - global _typing_module - if _typing_module is None: - typing_path = \ - os.path.abspath(os.path.join(__file__, "../jedi_typing.py")) - with open(typing_path) as f: - code = _compatibility.unicode(f.read()) - p = ParserWithRecovery(load_grammar(), code) - _typing_module = p.module - return _typing_module - - -def get_types_for_typing_module(evaluator, typ, node): - from jedi.evaluate.iterable import FakeSequence - if not typ.base.get_parent_until().name.value == "typing": - return None - # we assume that any class using [] in a module called - # "typing" with a name for which we have a replacement - # should be replaced by that class. This is not 100% - # airtight but I don't have a better idea to check that it's - # actually the PEP-0484 typing module and not some other - if tree.is_node(node, "subscriptlist"): - nodes = node.children[::2] # skip the commas - else: - nodes = [node] - del node - - nodes = [_fix_forward_reference(evaluator, node) for node in nodes] - - # hacked in Union and Optional, since it's hard to do nicely in parsed code - if typ.name.value == "Union": - return unite(evaluator.eval_element(node) for node in nodes) - if typ.name.value == "Optional": - return evaluator.eval_element(nodes[0]) - - typing = _get_typing_replacement_module() - factories = evaluator.find_types(typing, "factory") - assert len(factories) == 1 - factory = list(factories)[0] - assert factory - function_body_nodes = factory.children[4].children - valid_classnames = set(child.name.value - for child in function_body_nodes - if isinstance(child, tree.Class)) - if typ.name.value not in valid_classnames: - return None - compiled_classname = compiled.create(evaluator, typ.name.value) - - args = FakeSequence(evaluator, nodes, "tuple") - - result = evaluator.execute_evaluated(factory, compiled_classname, args) - return result - - -def find_type_from_comment_hint_for(evaluator, node, name): - return \ - _find_type_from_comment_hint(evaluator, node, node.children[1], name) - - -def find_type_from_comment_hint_with(evaluator, node, name): - assert len(node.children[1].children) == 3, \ - "Can only be here when children[1] is 'foo() as f'" - return _find_type_from_comment_hint( - evaluator, node, node.children[1].children[2], name) - - -def find_type_from_comment_hint_assign(evaluator, node, name): - return \ - _find_type_from_comment_hint(evaluator, node, node.children[0], name) - - -def _find_type_from_comment_hint(evaluator, node, varlist, name): - index = None - if varlist.type in ("testlist_star_expr", "exprlist"): - # something like "a, b = 1, 2" - index = 0 - for child in varlist.children: - if child == name: - break - if child.type == "operator": - continue - index += 1 - else: - return [] - - comment = node.get_following_comment_same_line() - if comment is None: - return [] - match = re.match(r"^#\s*type:\s*([^#]*)", comment) - if not match: - return [] - annotation = tree.String( - tree.zero_position_modifier, - repr(str(match.group(1).strip())), - node.start_pos) - annotation.parent = node.parent - return _evaluate_for_annotation(evaluator, annotation, index) diff --git a/pythonFiles/preview/jedi/evaluate/precedence.py b/pythonFiles/preview/jedi/evaluate/precedence.py deleted file mode 100644 index 5225aa6818ff..000000000000 --- a/pythonFiles/preview/jedi/evaluate/precedence.py +++ /dev/null @@ -1,178 +0,0 @@ -""" -Handles operator precedence. -""" -import operator - -from jedi._compatibility import unicode -from jedi.parser import tree -from jedi import debug -from jedi.evaluate.compiled import CompiledObject, create, builtin_from_name -from jedi.evaluate import analysis - -# Maps Python syntax to the operator module. -COMPARISON_OPERATORS = { - '==': operator.eq, - '!=': operator.ne, - 'is': operator.is_, - 'is not': operator.is_not, - '<': operator.lt, - '<=': operator.le, - '>': operator.gt, - '>=': operator.ge, -} - - -def literals_to_types(evaluator, result): - # Changes literals ('a', 1, 1.0, etc) to its type instances (str(), - # int(), float(), etc). - new_result = set() - for typ in result: - if is_literal(typ): - # Literals are only valid as long as the operations are - # correct. Otherwise add a value-free instance. - cls = builtin_from_name(evaluator, typ.name.value) - new_result |= evaluator.execute(cls) - else: - new_result.add(typ) - return new_result - - -def calculate_children(evaluator, children): - """ - Calculate a list of children with operators. - """ - iterator = iter(children) - types = evaluator.eval_element(next(iterator)) - for operator in iterator: - right = next(iterator) - if tree.is_node(operator, 'comp_op'): # not in / is not - operator = ' '.join(str(c.value) for c in operator.children) - - # handle lazy evaluation of and/or here. - if operator in ('and', 'or'): - left_bools = set([left.py__bool__() for left in types]) - if left_bools == set([True]): - if operator == 'and': - types = evaluator.eval_element(right) - elif left_bools == set([False]): - if operator != 'and': - types = evaluator.eval_element(right) - # Otherwise continue, because of uncertainty. - else: - types = calculate(evaluator, types, operator, - evaluator.eval_element(right)) - debug.dbg('calculate_children types %s', types) - return types - - -def calculate(evaluator, left_result, operator, right_result): - result = set() - if not left_result or not right_result: - # illegal slices e.g. cause left/right_result to be None - result = (left_result or set()) | (right_result or set()) - result = literals_to_types(evaluator, result) - else: - # I don't think there's a reasonable chance that a string - # operation is still correct, once we pass something like six - # objects. - if len(left_result) * len(right_result) > 6: - result = literals_to_types(evaluator, left_result | right_result) - else: - for left in left_result: - for right in right_result: - result |= _element_calculate(evaluator, left, operator, right) - return result - - -def factor_calculate(evaluator, types, operator): - """ - Calculates `+`, `-`, `~` and `not` prefixes. - """ - for typ in types: - if operator == '-': - if _is_number(typ): - yield create(evaluator, -typ.obj) - elif operator == 'not': - value = typ.py__bool__() - if value is None: # Uncertainty. - return - yield create(evaluator, not value) - else: - yield typ - - -def _is_number(obj): - return isinstance(obj, CompiledObject) \ - and isinstance(obj.obj, (int, float)) - - -def is_string(obj): - return isinstance(obj, CompiledObject) \ - and isinstance(obj.obj, (str, unicode)) - - -def is_literal(obj): - return _is_number(obj) or is_string(obj) - - -def _is_tuple(obj): - from jedi.evaluate import iterable - return isinstance(obj, iterable.Array) and obj.type == 'tuple' - - -def _is_list(obj): - from jedi.evaluate import iterable - return isinstance(obj, iterable.Array) and obj.type == 'list' - - -def _element_calculate(evaluator, left, operator, right): - from jedi.evaluate import iterable, representation as er - l_is_num = _is_number(left) - r_is_num = _is_number(right) - if operator == '*': - # for iterables, ignore * operations - if isinstance(left, iterable.Array) or is_string(left): - return set([left]) - elif isinstance(right, iterable.Array) or is_string(right): - return set([right]) - elif operator == '+': - if l_is_num and r_is_num or is_string(left) and is_string(right): - return set([create(evaluator, left.obj + right.obj)]) - elif _is_tuple(left) and _is_tuple(right) or _is_list(left) and _is_list(right): - return set([iterable.MergedArray(evaluator, (left, right))]) - elif operator == '-': - if l_is_num and r_is_num: - return set([create(evaluator, left.obj - right.obj)]) - elif operator == '%': - # With strings and numbers the left type typically remains. Except for - # `int() % float()`. - return set([left]) - elif operator in COMPARISON_OPERATORS: - operation = COMPARISON_OPERATORS[operator] - if isinstance(left, CompiledObject) and isinstance(right, CompiledObject): - # Possible, because the return is not an option. Just compare. - left = left.obj - right = right.obj - - try: - result = operation(left, right) - except TypeError: - # Could be True or False. - return set([create(evaluator, True), create(evaluator, False)]) - else: - return set([create(evaluator, result)]) - elif operator == 'in': - return set() - - def check(obj): - """Checks if a Jedi object is either a float or an int.""" - return isinstance(obj, er.Instance) and obj.name.get_code() in ('int', 'float') - - # Static analysis, one is a number, the other one is not. - if operator in ('+', '-') and l_is_num != r_is_num \ - and not (check(left) or check(right)): - message = "TypeError: unsupported operand type(s) for +: %s and %s" - analysis.add(evaluator, 'type-error-operation', operator, - message % (left, right)) - - return set([left, right]) diff --git a/pythonFiles/preview/jedi/evaluate/recursion.py b/pythonFiles/preview/jedi/evaluate/recursion.py deleted file mode 100644 index 9b95fad5adfd..000000000000 --- a/pythonFiles/preview/jedi/evaluate/recursion.py +++ /dev/null @@ -1,157 +0,0 @@ -""" -Recursions are the recipe of |jedi| to conquer Python code. However, someone -must stop recursions going mad. Some settings are here to make |jedi| stop at -the right time. You can read more about them :ref:`here `. - -Next to :mod:`jedi.evaluate.cache` this module also makes |jedi| not -thread-safe. Why? ``execution_recursion_decorator`` uses class variables to -count the function calls. -""" -from jedi import debug -from jedi import settings -from jedi.evaluate import iterable - - -def recursion_decorator(func): - def run(evaluator, stmt, *args, **kwargs): - rec_detect = evaluator.recursion_detector - if rec_detect.push_stmt(stmt): - return set() - else: - result = func(evaluator, stmt, *args, **kwargs) - rec_detect.pop_stmt() - return result - return run - - -class RecursionDetector(object): - """ - A decorator to detect recursions in statements. In a recursion a statement - at the same place, in the same module may not be executed two times. - """ - def __init__(self, evaluator): - self.top = None - self.current = None - self._evaluator = evaluator - - def push_stmt(self, stmt): - self.current = _RecursionNode(self._evaluator, stmt, self.current) - check = self._check_recursion() - if check: - debug.warning('catched stmt recursion: %s against %s @%s', stmt, - check.stmt, stmt.start_pos) - self.pop_stmt() - return True - return False - - def pop_stmt(self): - if self.current is not None: - # I don't know how current can be None, but sometimes it happens - # with Python3. - self.current = self.current.parent - - def _check_recursion(self): - test = self.current - while True: - test = test.parent - if self.current == test: - return test - if not test: - return False - - def node_statements(self): - result = [] - n = self.current - while n: - result.insert(0, n.stmt) - n = n.parent - return result - - -class _RecursionNode(object): - """ A node of the RecursionDecorator. """ - def __init__(self, evaluator, stmt, parent): - self._evaluator = evaluator - self.script = stmt.get_parent_until() - self.position = stmt.start_pos - self.parent = parent - self.stmt = stmt - - # Don't check param instances, they are not causing recursions - # The same's true for the builtins, because the builtins are really - # simple. - self.is_ignored = self.script == self._evaluator.BUILTINS - - def __eq__(self, other): - if not other: - return None - - return self.script == other.script \ - and self.position == other.position \ - and not self.is_ignored and not other.is_ignored - - -def execution_recursion_decorator(func): - def run(execution, **kwargs): - detector = execution._evaluator.execution_recursion_detector - if detector.push_execution(execution): - result = set() - else: - result = func(execution, **kwargs) - detector.pop_execution() - return result - - return run - - -class ExecutionRecursionDetector(object): - """ - Catches recursions of executions. - """ - def __init__(self, evaluator): - self.recursion_level = 0 - self.parent_execution_funcs = [] - self.execution_funcs = set() - self.execution_count = 0 - self._evaluator = evaluator - - def __call__(self, execution): - debug.dbg('Execution recursions: %s', execution, self.recursion_level, - self.execution_count, len(self.execution_funcs)) - if self.check_recursion(execution): - result = set() - else: - result = self.func(execution) - self.pop_execution() - return result - - def pop_execution(self): - self.parent_execution_funcs.pop() - self.recursion_level -= 1 - - def push_execution(self, execution): - in_par_execution_funcs = execution.base in self.parent_execution_funcs - in_execution_funcs = execution.base in self.execution_funcs - self.recursion_level += 1 - self.execution_count += 1 - self.execution_funcs.add(execution.base) - self.parent_execution_funcs.append(execution.base) - - if self.execution_count > settings.max_executions: - return True - - if isinstance(execution.base, (iterable.Array, iterable.Generator)): - return False - module = execution.get_parent_until() - if module == self._evaluator.BUILTINS: - return False - - if in_par_execution_funcs: - if self.recursion_level > settings.max_function_recursion_level: - return True - if in_execution_funcs and \ - len(self.execution_funcs) > settings.max_until_execution_unique: - return True - if self.execution_count > settings.max_executions_without_builtins: - return True - return False diff --git a/pythonFiles/preview/jedi/evaluate/representation.py b/pythonFiles/preview/jedi/evaluate/representation.py deleted file mode 100644 index 8f3d642cc762..000000000000 --- a/pythonFiles/preview/jedi/evaluate/representation.py +++ /dev/null @@ -1,974 +0,0 @@ -""" -Like described in the :mod:`jedi.parser.tree` module, -there's a need for an ast like module to represent the states of parsed -modules. - -But now there are also structures in Python that need a little bit more than -that. An ``Instance`` for example is only a ``Class`` before it is -instantiated. This class represents these cases. - -So, why is there also a ``Class`` class here? Well, there are decorators and -they change classes in Python 3. - -Representation modules also define "magic methods". Those methods look like -``py__foo__`` and are typically mappable to the Python equivalents ``__call__`` -and others. Here's a list: - -====================================== ======================================== -**Method** **Description** --------------------------------------- ---------------------------------------- -py__call__(params: Array) On callable objects, returns types. -py__bool__() Returns True/False/None; None means that - there's no certainty. -py__bases__() Returns a list of base classes. -py__mro__() Returns a list of classes (the mro). -py__iter__() Returns a generator of a set of types. -py__class__() Returns the class of an instance. -py__getitem__(index: int/str) Returns a a set of types of the index. - Can raise an IndexError/KeyError. -py__file__() Only on modules. -py__package__() Only on modules. For the import system. -py__path__() Only on modules. For the import system. -====================================== ======================================== - -__ -""" -import os -import pkgutil -import imp -import re -from itertools import chain - -from jedi._compatibility import use_metaclass, unicode, Python3Method, is_py3 -from jedi.parser import tree -from jedi import debug -from jedi import common -from jedi.cache import underscore_memoization, cache_star_import -from jedi.evaluate.cache import memoize_default, CachedMetaClass, NO_DEFAULT -from jedi.evaluate import compiled -from jedi.evaluate.compiled import mixed -from jedi.evaluate import recursion -from jedi.evaluate import iterable -from jedi.evaluate import docstrings -from jedi.evaluate import pep0484 -from jedi.evaluate import helpers -from jedi.evaluate import param -from jedi.evaluate import flow_analysis -from jedi.evaluate import imports - - -class Executed(tree.Base): - """ - An instance is also an executable - because __init__ is called - :param var_args: The param input array, consist of a parser node or a list. - """ - def __init__(self, evaluator, base, var_args=()): - self._evaluator = evaluator - self.base = base - self.var_args = var_args - - def is_scope(self): - return True - - def get_parent_until(self, *args, **kwargs): - return tree.Base.get_parent_until(self, *args, **kwargs) - - @common.safe_property - def parent(self): - return self.base.parent - - -class Instance(use_metaclass(CachedMetaClass, Executed)): - """ - This class is used to evaluate instances. - """ - def __init__(self, evaluator, base, var_args, is_generated=False): - super(Instance, self).__init__(evaluator, base, var_args) - self.decorates = None - # Generated instances are classes that are just generated by self - # (No var_args) used. - self.is_generated = is_generated - - if base.name.get_code() in ['list', 'set'] \ - and evaluator.BUILTINS == base.get_parent_until(): - # compare the module path with the builtin name. - self.var_args = iterable.check_array_instances(evaluator, self) - elif not is_generated: - # Need to execute the __init__ function, because the dynamic param - # searching needs it. - try: - method = self.get_subscope_by_name('__init__') - except KeyError: - pass - else: - evaluator.execute(method, self.var_args) - - def is_class(self): - return False - - @property - def py__call__(self): - def actual(params): - return self._evaluator.execute(method, params) - - try: - method = self.get_subscope_by_name('__call__') - except KeyError: - # Means the Instance is not callable. - raise AttributeError - - return actual - - def py__class__(self): - return self.base - - def py__bool__(self): - # Signalize that we don't know about the bool type. - return None - - @memoize_default() - def _get_method_execution(self, func): - func = get_instance_el(self._evaluator, self, func, True) - return FunctionExecution(self._evaluator, func, self.var_args) - - def _get_func_self_name(self, func): - """ - Returns the name of the first param in a class method (which is - normally self. - """ - try: - return str(func.params[0].name) - except IndexError: - return None - - def _self_names_dict(self, add_mro=True): - names = {} - # This loop adds the names of the self object, copies them and removes - # the self. - for sub in self.base.subscopes: - if isinstance(sub, tree.Class): - continue - # Get the self name, if there's one. - self_name = self._get_func_self_name(sub) - if self_name is None: - continue - - if sub.name.value == '__init__' and not self.is_generated: - # ``__init__`` is special because the params need are injected - # this way. Therefore an execution is necessary. - if not sub.get_decorators(): - # __init__ decorators should generally just be ignored, - # because to follow them and their self variables is too - # complicated. - sub = self._get_method_execution(sub) - for name_list in sub.names_dict.values(): - for name in name_list: - if name.value == self_name and name.get_previous_sibling() is None: - trailer = name.get_next_sibling() - if tree.is_node(trailer, 'trailer') \ - and len(trailer.children) == 2 \ - and trailer.children[0] == '.': - name = trailer.children[1] # After dot. - if name.is_definition(): - arr = names.setdefault(name.value, []) - arr.append(get_instance_el(self._evaluator, self, name)) - return names - - def get_subscope_by_name(self, name): - sub = self.base.get_subscope_by_name(name) - return get_instance_el(self._evaluator, self, sub, True) - - def execute_subscope_by_name(self, name, *args): - method = self.get_subscope_by_name(name) - return self._evaluator.execute_evaluated(method, *args) - - def get_descriptor_returns(self, obj): - """ Throws a KeyError if there's no method. """ - # Arguments in __get__ descriptors are obj, class. - # `method` is the new parent of the array, don't know if that's good. - none_obj = compiled.create(self._evaluator, None) - args = [obj, obj.base] if isinstance(obj, Instance) else [none_obj, obj] - try: - return self.execute_subscope_by_name('__get__', *args) - except KeyError: - return set([self]) - - @memoize_default() - def names_dicts(self, search_global): - yield self._self_names_dict() - - for s in self.base.py__mro__()[1:]: - if not isinstance(s, compiled.CompiledObject): - # Compiled objects don't have `self.` names. - for inst in self._evaluator.execute(s): - yield inst._self_names_dict(add_mro=False) - - for names_dict in self.base.names_dicts(search_global=False, is_instance=True): - yield LazyInstanceDict(self._evaluator, self, names_dict) - - def py__getitem__(self, index): - try: - method = self.get_subscope_by_name('__getitem__') - except KeyError: - debug.warning('No __getitem__, cannot access the array.') - return set() - else: - index_obj = compiled.create(self._evaluator, index) - return self._evaluator.execute_evaluated(method, index_obj) - - def py__iter__(self): - try: - method = self.get_subscope_by_name('__iter__') - except KeyError: - debug.warning('No __iter__ on %s.' % self) - return - else: - iters = self._evaluator.execute(method) - for generator in iters: - if isinstance(generator, Instance): - # `__next__` logic. - name = '__next__' if is_py3 else 'next' - try: - yield generator.execute_subscope_by_name(name) - except KeyError: - debug.warning('Instance has no __next__ function in %s.', generator) - else: - for typ in generator.py__iter__(): - yield typ - - @property - @underscore_memoization - def name(self): - name = self.base.name - return helpers.FakeName(unicode(name), self, name.start_pos) - - def __getattr__(self, name): - if name not in ['start_pos', 'end_pos', 'get_imports', 'type', - 'doc', 'raw_doc']: - raise AttributeError("Instance %s: Don't touch this (%s)!" - % (self, name)) - return getattr(self.base, name) - - def __repr__(self): - dec = '' - if self.decorates is not None: - dec = " decorates " + repr(self.decorates) - return "<%s of %s(%s)%s>" % (type(self).__name__, self.base, - self.var_args, dec) - - -class LazyInstanceDict(object): - def __init__(self, evaluator, instance, dct): - self._evaluator = evaluator - self._instance = instance - self._dct = dct - - def __getitem__(self, name): - return [get_instance_el(self._evaluator, self._instance, var, True) - for var in self._dct[name]] - - def values(self): - return [self[key] for key in self._dct] - - -class InstanceName(tree.Name): - def __init__(self, origin_name, parent): - super(InstanceName, self).__init__(tree.zero_position_modifier, - origin_name.value, - origin_name.start_pos) - self._origin_name = origin_name - self.parent = parent - - def is_definition(self): - return self._origin_name.is_definition() - - -def get_instance_el(evaluator, instance, var, is_class_var=False): - """ - Returns an InstanceElement if it makes sense, otherwise leaves the object - untouched. - - Basically having an InstanceElement is context information. That is needed - in quite a lot of cases, which includes Nodes like ``power``, that need to - know where a self name comes from for example. - """ - if isinstance(var, tree.Name): - parent = get_instance_el(evaluator, instance, var.parent, is_class_var) - return InstanceName(var, parent) - elif var.type != 'funcdef' \ - and isinstance(var, (Instance, compiled.CompiledObject, tree.Leaf, - tree.Module, FunctionExecution)): - return var - - var = evaluator.wrap(var) - return InstanceElement(evaluator, instance, var, is_class_var) - - -class InstanceElement(use_metaclass(CachedMetaClass, tree.Base)): - """ - InstanceElement is a wrapper for any object, that is used as an instance - variable (e.g. self.variable or class methods). - """ - def __init__(self, evaluator, instance, var, is_class_var): - self._evaluator = evaluator - self.instance = instance - self.var = var - self.is_class_var = is_class_var - - @common.safe_property - @memoize_default() - def parent(self): - par = self.var.parent - if isinstance(par, Class) and par == self.instance.base \ - or isinstance(par, tree.Class) \ - and par == self.instance.base.base: - par = self.instance - else: - par = get_instance_el(self._evaluator, self.instance, par, - self.is_class_var) - return par - - def get_parent_until(self, *args, **kwargs): - return tree.BaseNode.get_parent_until(self, *args, **kwargs) - - def get_definition(self): - return self.get_parent_until((tree.ExprStmt, tree.IsScope, tree.Import)) - - def get_decorated_func(self): - """ Needed because the InstanceElement should not be stripped """ - func = self.var.get_decorated_func() - func = get_instance_el(self._evaluator, self.instance, func) - return func - - def get_rhs(self): - return get_instance_el(self._evaluator, self.instance, - self.var.get_rhs(), self.is_class_var) - - def is_definition(self): - return self.var.is_definition() - - @property - def children(self): - # Copy and modify the array. - return [get_instance_el(self._evaluator, self.instance, command, self.is_class_var) - for command in self.var.children] - - @property - @memoize_default() - def name(self): - name = self.var.name - return helpers.FakeName(unicode(name), self, name.start_pos) - - def __iter__(self): - for el in self.var.__iter__(): - yield get_instance_el(self._evaluator, self.instance, el, - self.is_class_var) - - def __getitem__(self, index): - return get_instance_el(self._evaluator, self.instance, self.var[index], - self.is_class_var) - - def __getattr__(self, name): - return getattr(self.var, name) - - def isinstance(self, *cls): - return isinstance(self.var, cls) - - def is_scope(self): - """ - Since we inherit from Base, it would overwrite the action we want here. - """ - return self.var.is_scope() - - def py__call__(self, params): - if isinstance(self.var, compiled.CompiledObject): - # This check is a bit strange, but CompiledObject itself is a bit - # more complicated than we would it actually like to be. - return self.var.py__call__(params) - else: - return Function.py__call__(self, params) - - def __repr__(self): - return "<%s of %s>" % (type(self).__name__, self.var) - - -class Wrapper(tree.Base): - def is_scope(self): - return True - - def is_class(self): - return False - - def py__bool__(self): - """ - Since Wrapper is a super class for classes, functions and modules, - the return value will always be true. - """ - return True - - @property - @underscore_memoization - def name(self): - name = self.base.name - return helpers.FakeName(unicode(name), self, name.start_pos) - - -class Class(use_metaclass(CachedMetaClass, Wrapper)): - """ - This class is not only important to extend `tree.Class`, it is also a - important for descriptors (if the descriptor methods are evaluated or not). - """ - def __init__(self, evaluator, base): - self._evaluator = evaluator - self.base = base - - @memoize_default(default=()) - def py__mro__(self): - def add(cls): - if cls not in mro: - mro.append(cls) - - mro = [self] - # TODO Do a proper mro resolution. Currently we are just listing - # classes. However, it's a complicated algorithm. - for cls in self.py__bases__(): - # TODO detect for TypeError: duplicate base class str, - # e.g. `class X(str, str): pass` - try: - mro_method = cls.py__mro__ - except AttributeError: - # TODO add a TypeError like: - """ - >>> class Y(lambda: test): pass - Traceback (most recent call last): - File "", line 1, in - TypeError: function() argument 1 must be code, not str - >>> class Y(1): pass - Traceback (most recent call last): - File "", line 1, in - TypeError: int() takes at most 2 arguments (3 given) - """ - pass - else: - add(cls) - for cls_new in mro_method(): - add(cls_new) - return tuple(mro) - - @memoize_default(default=()) - def py__bases__(self): - arglist = self.base.get_super_arglist() - if arglist: - args = param.Arguments(self._evaluator, arglist) - return list(chain.from_iterable(args.eval_args())) - else: - return [compiled.create(self._evaluator, object)] - - def py__call__(self, params): - return set([Instance(self._evaluator, self, params)]) - - def py__class__(self): - return compiled.create(self._evaluator, type) - - @property - def params(self): - try: - return self.get_subscope_by_name('__init__').params - except KeyError: - return [] # object.__init__ - - def names_dicts(self, search_global, is_instance=False): - if search_global: - yield self.names_dict - else: - for scope in self.py__mro__(): - if isinstance(scope, compiled.CompiledObject): - yield scope.names_dicts(False, is_instance)[0] - else: - yield scope.names_dict - - def is_class(self): - return True - - def get_subscope_by_name(self, name): - for s in self.py__mro__(): - for sub in reversed(s.subscopes): - if sub.name.value == name: - return sub - raise KeyError("Couldn't find subscope.") - - def __getattr__(self, name): - if name not in ['start_pos', 'end_pos', 'parent', 'raw_doc', - 'doc', 'get_imports', 'get_parent_until', 'get_code', - 'subscopes', 'names_dict', 'type']: - raise AttributeError("Don't touch this: %s of %s !" % (name, self)) - return getattr(self.base, name) - - def __repr__(self): - return "" % (type(self).__name__, self.base) - - -class Function(use_metaclass(CachedMetaClass, Wrapper)): - """ - Needed because of decorators. Decorators are evaluated here. - """ - def __init__(self, evaluator, func, is_decorated=False): - """ This should not be called directly """ - self._evaluator = evaluator - self.base = self.base_func = func - self.is_decorated = is_decorated - # A property that is set by the decorator resolution. - self.decorates = None - - @memoize_default() - def get_decorated_func(self): - """ - Returns the function, that should to be executed in the end. - This is also the places where the decorators are processed. - """ - f = self.base_func - decorators = self.base_func.get_decorators() - - if not decorators or self.is_decorated: - return self - - # Only enter it, if has not already been processed. - if not self.is_decorated: - for dec in reversed(decorators): - debug.dbg('decorator: %s %s', dec, f) - dec_results = self._evaluator.eval_element(dec.children[1]) - trailer = dec.children[2:-1] - if trailer: - # Create a trailer and evaluate it. - trailer = tree.Node('trailer', trailer) - trailer.parent = dec - dec_results = self._evaluator.eval_trailer(dec_results, trailer) - - if not len(dec_results): - debug.warning('decorator not found: %s on %s', dec, self.base_func) - return self - decorator = dec_results.pop() - if dec_results: - debug.warning('multiple decorators found %s %s', - self.base_func, dec_results) - - # Create param array. - if isinstance(f, Function): - old_func = f # TODO this is just hacky. change. - elif f.type == 'funcdef': - old_func = Function(self._evaluator, f, is_decorated=True) - else: - old_func = f - - wrappers = self._evaluator.execute_evaluated(decorator, old_func) - if not len(wrappers): - debug.warning('no wrappers found %s', self.base_func) - return self - if len(wrappers) > 1: - # TODO resolve issue with multiple wrappers -> multiple types - debug.warning('multiple wrappers found %s %s', - self.base_func, wrappers) - f = list(wrappers)[0] - if isinstance(f, (Instance, Function)): - f.decorates = self - - debug.dbg('decorator end %s', f) - return f - - def names_dicts(self, search_global): - if search_global: - yield self.names_dict - else: - scope = self.py__class__() - for names_dict in scope.names_dicts(False): - yield names_dict - - @Python3Method - def py__call__(self, params): - if self.base.is_generator(): - return set([iterable.Generator(self._evaluator, self, params)]) - else: - return FunctionExecution(self._evaluator, self, params).get_return_types() - - @memoize_default() - def py__annotations__(self): - parser_func = self.base - return_annotation = parser_func.annotation() - if return_annotation: - dct = {'return': return_annotation} - else: - dct = {} - for function_param in parser_func.params: - param_annotation = function_param.annotation() - if param_annotation is not None: - dct[function_param.name.value] = param_annotation - return dct - - def py__class__(self): - # This differentiation is only necessary for Python2. Python3 does not - # use a different method class. - if isinstance(self.base.get_parent_scope(), tree.Class): - name = 'METHOD_CLASS' - else: - name = 'FUNCTION_CLASS' - return compiled.get_special_object(self._evaluator, name) - - def __getattr__(self, name): - return getattr(self.base_func, name) - - def __repr__(self): - dec = '' - if self.decorates is not None: - dec = " decorates " + repr(self.decorates) - return "" % (type(self).__name__, self.base_func, dec) - - -class LambdaWrapper(Function): - def get_decorated_func(self): - return self - - -class FunctionExecution(Executed): - """ - This class is used to evaluate functions and their returns. - - This is the most complicated class, because it contains the logic to - transfer parameters. It is even more complicated, because there may be - multiple calls to functions and recursion has to be avoided. But this is - responsibility of the decorators. - """ - type = 'funcdef' - - def __init__(self, evaluator, base, *args, **kwargs): - super(FunctionExecution, self).__init__(evaluator, base, *args, **kwargs) - self._copy_dict = {} - funcdef = base.base_func - if isinstance(funcdef, mixed.MixedObject): - # The extra information in mixed is not needed anymore. We can just - # unpack it and give it the tree object. - funcdef = funcdef.definition - - # Just overwrite the old version. We don't need it anymore. - funcdef = helpers.deep_ast_copy(funcdef, new_elements=self._copy_dict) - for child in funcdef.children: - if child.type not in ('operator', 'keyword'): - # Not all nodes are properly copied by deep_ast_copy. - child.parent = self - self.children = funcdef.children - self.names_dict = funcdef.names_dict - - @memoize_default(default=set()) - @recursion.execution_recursion_decorator - def get_return_types(self, check_yields=False): - func = self.base - - if func.isinstance(LambdaWrapper): - return self._evaluator.eval_element(self.children[-1]) - - if func.listeners: - # Feed the listeners, with the params. - for listener in func.listeners: - listener.execute(self._get_params()) - # If we do have listeners, that means that there's not a regular - # execution ongoing. In this case Jedi is interested in the - # inserted params, not in the actual execution of the function. - return set() - - if check_yields: - types = set() - returns = self.yields - else: - returns = self.returns - types = set(docstrings.find_return_types(self._evaluator, func)) - types |= set(pep0484.find_return_types(self._evaluator, func)) - - for r in returns: - check = flow_analysis.break_check(self._evaluator, self, r) - if check is flow_analysis.UNREACHABLE: - debug.dbg('Return unreachable: %s', r) - else: - if check_yields: - types |= iterable.unite(self._eval_yield(r)) - else: - types |= self._evaluator.eval_element(r.children[1]) - if check is flow_analysis.REACHABLE: - debug.dbg('Return reachable: %s', r) - break - return types - - def _eval_yield(self, yield_expr): - element = yield_expr.children[1] - if element.type == 'yield_arg': - # It must be a yield from. - yield_from_types = self._evaluator.eval_element(element.children[1]) - for result in iterable.py__iter__(self._evaluator, yield_from_types, element): - yield result - else: - yield self._evaluator.eval_element(element) - - @recursion.execution_recursion_decorator - def get_yield_types(self): - yields = self.yields - stopAt = tree.ForStmt, tree.WhileStmt, FunctionExecution, tree.IfStmt - for_parents = [(x, x.get_parent_until((stopAt))) for x in yields] - - # Calculate if the yields are placed within the same for loop. - yields_order = [] - last_for_stmt = None - for yield_, for_stmt in for_parents: - # For really simple for loops we can predict the order. Otherwise - # we just ignore it. - parent = for_stmt.parent - if parent.type == 'suite': - parent = parent.parent - if for_stmt.type == 'for_stmt' and parent == self \ - and for_stmt.defines_one_name(): # Simplicity for now. - if for_stmt == last_for_stmt: - yields_order[-1][1].append(yield_) - else: - yields_order.append((for_stmt, [yield_])) - elif for_stmt == self: - yields_order.append((None, [yield_])) - else: - yield self.get_return_types(check_yields=True) - return - last_for_stmt = for_stmt - - evaluator = self._evaluator - for for_stmt, yields in yields_order: - if for_stmt is None: - # No for_stmt, just normal yields. - for yield_ in yields: - for result in self._eval_yield(yield_): - yield result - else: - input_node = for_stmt.get_input_node() - for_types = evaluator.eval_element(input_node) - ordered = iterable.py__iter__(evaluator, for_types, input_node) - for index_types in ordered: - dct = {str(for_stmt.children[1]): index_types} - evaluator.predefined_if_name_dict_dict[for_stmt] = dct - for yield_in_same_for_stmt in yields: - for result in self._eval_yield(yield_in_same_for_stmt): - yield result - del evaluator.predefined_if_name_dict_dict[for_stmt] - - def names_dicts(self, search_global): - yield self.names_dict - - @memoize_default(default=NO_DEFAULT) - def _get_params(self): - """ - This returns the params for an TODO and is injected as a - 'hack' into the tree.Function class. - This needs to be here, because Instance can have __init__ functions, - which act the same way as normal functions. - """ - return param.get_params(self._evaluator, self.base, self.var_args) - - def param_by_name(self, name): - return [n for n in self._get_params() if str(n) == name][0] - - def name_for_position(self, position): - return tree.Function.name_for_position(self, position) - - def __getattr__(self, name): - if name not in ['start_pos', 'end_pos', 'imports', 'name', 'type']: - raise AttributeError('Tried to access %s: %s. Why?' % (name, self)) - return getattr(self.base, name) - - @common.safe_property - @memoize_default() - def returns(self): - return tree.Scope._search_in_scope(self, tree.ReturnStmt) - - @common.safe_property - @memoize_default() - def yields(self): - return tree.Scope._search_in_scope(self, tree.YieldExpr) - - @common.safe_property - @memoize_default() - def statements(self): - return tree.Scope._search_in_scope(self, tree.ExprStmt) - - @common.safe_property - @memoize_default() - def subscopes(self): - return tree.Scope._search_in_scope(self, tree.Scope) - - def __repr__(self): - return "<%s of %s>" % (type(self).__name__, self.base) - - -class GlobalName(helpers.FakeName): - def __init__(self, name): - """ - We need to mark global names somehow. Otherwise they are just normal - names that are not definitions. - """ - super(GlobalName, self).__init__(name.value, name.parent, - name.start_pos, is_definition=True) - - -class ModuleWrapper(use_metaclass(CachedMetaClass, tree.Module, Wrapper)): - def __init__(self, evaluator, module, parent_module=None): - self._evaluator = evaluator - self.base = self._module = module - self._parent_module = parent_module - - def names_dicts(self, search_global): - yield self.base.names_dict - yield self._module_attributes_dict() - - for star_module in self.star_imports(): - yield star_module.names_dict - - yield dict((str(n), [GlobalName(n)]) for n in self.base.global_names) - yield self._sub_modules_dict() - - # I'm not sure if the star import cache is really that effective anymore - # with all the other really fast import caches. Recheck. Also we would need - # to push the star imports into Evaluator.modules, if we reenable this. - #@cache_star_import - @memoize_default([]) - def star_imports(self): - modules = [] - for i in self.base.imports: - if i.is_star_import(): - name = i.star_import_name() - new = imports.ImportWrapper(self._evaluator, name).follow() - for module in new: - if isinstance(module, tree.Module): - modules += module.star_imports() - modules += new - return modules - - @memoize_default() - def _module_attributes_dict(self): - def parent_callback(): - # Create a string type object (without a defined string in it): - return list(self._evaluator.execute(compiled.create(self._evaluator, str)))[0] - - names = ['__file__', '__package__', '__doc__', '__name__'] - # All the additional module attributes are strings. - return dict((n, [helpers.LazyName(n, parent_callback, is_definition=True)]) - for n in names) - - @property - @memoize_default() - def name(self): - return helpers.FakeName(unicode(self.base.name), self, (1, 0)) - - def _get_init_directory(self): - """ - :return: The path to the directory of a package. None in case it's not - a package. - """ - for suffix, _, _ in imp.get_suffixes(): - ending = '__init__' + suffix - py__file__ = self.py__file__() - if py__file__ is not None and py__file__.endswith(ending): - # Remove the ending, including the separator. - return self.py__file__()[:-len(ending) - 1] - return None - - def py__name__(self): - for name, module in self._evaluator.modules.items(): - if module == self: - return name - - return '__main__' - - def py__file__(self): - """ - In contrast to Python's __file__ can be None. - """ - if self._module.path is None: - return None - - return os.path.abspath(self._module.path) - - def py__package__(self): - if self._get_init_directory() is None: - return re.sub(r'\.?[^\.]+$', '', self.py__name__()) - else: - return self.py__name__() - - def _py__path__(self): - if self._parent_module is None: - search_path = self._evaluator.sys_path - else: - search_path = self._parent_module.py__path__() - init_path = self.py__file__() - if os.path.basename(init_path) == '__init__.py': - with open(init_path, 'rb') as f: - content = common.source_to_unicode(f.read()) - # these are strings that need to be used for namespace packages, - # the first one is ``pkgutil``, the second ``pkg_resources``. - options = ('declare_namespace(__name__)', 'extend_path(__path__') - if options[0] in content or options[1] in content: - # It is a namespace, now try to find the rest of the - # modules on sys_path or whatever the search_path is. - paths = set() - for s in search_path: - other = os.path.join(s, unicode(self.name)) - if os.path.isdir(other): - paths.add(other) - return list(paths) - # Default to this. - return [self._get_init_directory()] - - @property - def py__path__(self): - """ - Not seen here, since it's a property. The callback actually uses a - variable, so use it like:: - - foo.py__path__(sys_path) - - In case of a package, this returns Python's __path__ attribute, which - is a list of paths (strings). - Raises an AttributeError if the module is not a package. - """ - path = self._get_init_directory() - - if path is None: - raise AttributeError('Only packages have __path__ attributes.') - else: - return self._py__path__ - - @memoize_default() - def _sub_modules_dict(self): - """ - Lists modules in the directory of this module (if this module is a - package). - """ - path = self._module.path - names = {} - if path is not None and path.endswith(os.path.sep + '__init__.py'): - mods = pkgutil.iter_modules([os.path.dirname(path)]) - for module_loader, name, is_pkg in mods: - fake_n = helpers.FakeName(name) - # It's obviously a relative import to the current module. - imp = helpers.FakeImport(fake_n, self, level=1) - fake_n.parent = imp - names[name] = [fake_n] - - # TODO add something like this in the future, its cleaner than the - # import hacks. - # ``os.path`` is a hardcoded exception, because it's a - # ``sys.modules`` modification. - #if str(self.name) == 'os': - # names.append(helpers.FakeName('path', parent=self)) - - return names - - def py__class__(self): - return compiled.get_special_object(self._evaluator, 'MODULE_CLASS') - - def __getattr__(self, name): - return getattr(self._module, name) - - def __repr__(self): - return "<%s: %s>" % (type(self).__name__, self._module) diff --git a/pythonFiles/preview/jedi/evaluate/stdlib.py b/pythonFiles/preview/jedi/evaluate/stdlib.py deleted file mode 100644 index f0c4346d6852..000000000000 --- a/pythonFiles/preview/jedi/evaluate/stdlib.py +++ /dev/null @@ -1,280 +0,0 @@ -""" -Implementations of standard library functions, because it's not possible to -understand them with Jedi. - -To add a new implementation, create a function and add it to the -``_implemented`` dict at the bottom of this module. - -Note that this module exists only to implement very specific functionality in -the standard library. The usual way to understand the standard library is the -compiled module that returns the types for C-builtins. -""" -import collections -import re - -from jedi._compatibility import unicode -from jedi.common import unite -from jedi.evaluate import compiled -from jedi.evaluate import representation as er -from jedi.evaluate import iterable -from jedi.parser import ParserWithRecovery -from jedi.parser import tree -from jedi import debug -from jedi.evaluate import precedence -from jedi.evaluate import param -from jedi.evaluate import analysis - - -class NotInStdLib(LookupError): - pass - - -def execute(evaluator, obj, arguments): - try: - obj_name = str(obj.name) - except AttributeError: - pass - else: - if obj.parent == evaluator.BUILTINS: - module_name = 'builtins' - elif isinstance(obj.parent, tree.Module): - module_name = str(obj.parent.name) - else: - module_name = '' - - # for now we just support builtin functions. - try: - func = _implemented[module_name][obj_name] - except KeyError: - pass - else: - return func(evaluator, obj, arguments) - raise NotInStdLib() - - -def _follow_param(evaluator, arguments, index): - try: - key, values = list(arguments.unpack())[index] - except IndexError: - return set() - else: - return unite(evaluator.eval_element(v) for v in values) - - -def argument_clinic(string, want_obj=False, want_scope=False, want_arguments=False): - """ - Works like Argument Clinic (PEP 436), to validate function params. - """ - clinic_args = [] - allow_kwargs = False - optional = False - while string: - # Optional arguments have to begin with a bracket. And should always be - # at the end of the arguments. This is therefore not a proper argument - # clinic implementation. `range()` for exmple allows an optional start - # value at the beginning. - match = re.match('(?:(?:(\[),? ?|, ?|)(\w+)|, ?/)\]*', string) - string = string[len(match.group(0)):] - if not match.group(2): # A slash -> allow named arguments - allow_kwargs = True - continue - optional = optional or bool(match.group(1)) - word = match.group(2) - clinic_args.append((word, optional, allow_kwargs)) - - def f(func): - def wrapper(evaluator, obj, arguments): - debug.dbg('builtin start %s' % obj, color='MAGENTA') - try: - lst = list(arguments.eval_argument_clinic(clinic_args)) - except ValueError: - return set() - else: - kwargs = {} - if want_scope: - kwargs['scope'] = arguments.scope() - if want_obj: - kwargs['obj'] = obj - if want_arguments: - kwargs['arguments'] = arguments - return func(evaluator, *lst, **kwargs) - finally: - debug.dbg('builtin end', color='MAGENTA') - - return wrapper - return f - - -@argument_clinic('object, name[, default], /') -def builtins_getattr(evaluator, objects, names, defaults=None): - # follow the first param - for obj in objects: - if not isinstance(obj, (er.Instance, er.Class, tree.Module, compiled.CompiledObject)): - debug.warning('getattr called without instance') - continue - - for name in names: - if precedence.is_string(name): - return evaluator.find_types(obj, name.obj) - else: - debug.warning('getattr called without str') - continue - return set() - - -@argument_clinic('object[, bases, dict], /') -def builtins_type(evaluator, objects, bases, dicts): - if bases or dicts: - # It's a type creation... maybe someday... - return set() - else: - return set([o.py__class__() for o in objects]) - - -class SuperInstance(er.Instance): - """To be used like the object ``super`` returns.""" - def __init__(self, evaluator, cls): - su = cls.py_mro()[1] - super().__init__(evaluator, su and su[0] or self) - - -@argument_clinic('[type[, obj]], /', want_scope=True) -def builtins_super(evaluator, types, objects, scope): - # TODO make this able to detect multiple inheritance super - accept = (tree.Function, er.FunctionExecution) - if scope.isinstance(*accept): - wanted = (tree.Class, er.Instance) - cls = scope.get_parent_until(accept + wanted, - include_current=False) - if isinstance(cls, wanted): - if isinstance(cls, tree.Class): - cls = er.Class(evaluator, cls) - elif isinstance(cls, er.Instance): - cls = cls.base - su = cls.py__bases__() - if su: - return evaluator.execute(su[0]) - return set() - - -@argument_clinic('sequence, /', want_obj=True, want_arguments=True) -def builtins_reversed(evaluator, sequences, obj, arguments): - # While we could do without this variable (just by using sequences), we - # want static analysis to work well. Therefore we need to generated the - # values again. - first_arg = next(arguments.as_tuple())[0] - ordered = list(iterable.py__iter__(evaluator, sequences, first_arg)) - - rev = [iterable.AlreadyEvaluated(o) for o in reversed(ordered)] - # Repack iterator values and then run it the normal way. This is - # necessary, because `reversed` is a function and autocompletion - # would fail in certain cases like `reversed(x).__iter__` if we - # just returned the result directly. - rev = iterable.AlreadyEvaluated( - [iterable.FakeSequence(evaluator, rev, 'list')] - ) - return set([er.Instance(evaluator, obj, param.Arguments(evaluator, [rev]))]) - - -@argument_clinic('obj, type, /', want_arguments=True) -def builtins_isinstance(evaluator, objects, types, arguments): - bool_results = set([]) - for o in objects: - try: - mro_func = o.py__class__().py__mro__ - except AttributeError: - # This is temporary. Everything should have a class attribute in - # Python?! Maybe we'll leave it here, because some numpy objects or - # whatever might not. - return set([compiled.create(True), compiled.create(False)]) - - mro = mro_func() - - for cls_or_tup in types: - if cls_or_tup.is_class(): - bool_results.add(cls_or_tup in mro) - elif str(cls_or_tup.name) == 'tuple' \ - and cls_or_tup.get_parent_scope() == evaluator.BUILTINS: - # Check for tuples. - classes = unite(cls_or_tup.py__iter__()) - bool_results.add(any(cls in mro for cls in classes)) - else: - _, nodes = list(arguments.unpack())[1] - for node in nodes: - message = 'TypeError: isinstance() arg 2 must be a ' \ - 'class, type, or tuple of classes and types, ' \ - 'not %s.' % cls_or_tup - analysis.add(evaluator, 'type-error-isinstance', node, message) - - return set(compiled.create(evaluator, x) for x in bool_results) - - -def collections_namedtuple(evaluator, obj, arguments): - """ - Implementation of the namedtuple function. - - This has to be done by processing the namedtuple class template and - evaluating the result. - - .. note:: |jedi| only supports namedtuples on Python >2.6. - - """ - # Namedtuples are not supported on Python 2.6 - if not hasattr(collections, '_class_template'): - return set() - - # Process arguments - # TODO here we only use one of the types, we should use all. - name = list(_follow_param(evaluator, arguments, 0))[0].obj - _fields = list(_follow_param(evaluator, arguments, 1))[0] - if isinstance(_fields, compiled.CompiledObject): - fields = _fields.obj.replace(',', ' ').split() - elif isinstance(_fields, iterable.Array): - try: - fields = [v.obj for v in unite(_fields.py__iter__())] - except AttributeError: - return set() - else: - return set() - - # Build source - source = collections._class_template.format( - typename=name, - field_names=fields, - num_fields=len(fields), - arg_list=', '.join(fields), - repr_fmt=', '.join(collections._repr_template.format(name=name) for name in fields), - field_defs='\n'.join(collections._field_template.format(index=index, name=name) - for index, name in enumerate(fields)) - ) - - # Parse source - generated_class = ParserWithRecovery(evaluator.grammar, unicode(source)).module.subscopes[0] - return set([er.Class(evaluator, generated_class)]) - - -@argument_clinic('first, /') -def _return_first_param(evaluator, firsts): - return firsts - - -_implemented = { - 'builtins': { - 'getattr': builtins_getattr, - 'type': builtins_type, - 'super': builtins_super, - 'reversed': builtins_reversed, - 'isinstance': builtins_isinstance, - }, - 'copy': { - 'copy': _return_first_param, - 'deepcopy': _return_first_param, - }, - 'json': { - 'load': lambda *args: set(), - 'loads': lambda *args: set(), - }, - 'collections': { - 'namedtuple': collections_namedtuple, - }, -} diff --git a/pythonFiles/preview/jedi/evaluate/sys_path.py b/pythonFiles/preview/jedi/evaluate/sys_path.py deleted file mode 100644 index 4fbc6acacdc5..000000000000 --- a/pythonFiles/preview/jedi/evaluate/sys_path.py +++ /dev/null @@ -1,283 +0,0 @@ -import glob -import os -import sys -from jedi.evaluate.site import addsitedir - -from jedi._compatibility import exec_function, unicode -from jedi.parser import tree -from jedi.parser import ParserWithRecovery -from jedi.evaluate.cache import memoize_default -from jedi import debug -from jedi import common -from jedi.parser.utils import load_parser, save_parser - - -def get_venv_path(venv): - """Get sys.path for specified virtual environment.""" - sys_path = _get_venv_path_dirs(venv) - with common.ignored(ValueError): - sys_path.remove('') - sys_path = _get_sys_path_with_egglinks(sys_path) - # As of now, get_venv_path_dirs does not scan built-in pythonpath and - # user-local site-packages, let's approximate them using path from Jedi - # interpreter. - return sys_path + sys.path - - -def _get_sys_path_with_egglinks(sys_path): - """Find all paths including those referenced by egg-links. - - Egg-link-referenced directories are inserted into path immediately before - the directory on which their links were found. Such directories are not - taken into consideration by normal import mechanism, but they are traversed - when doing pkg_resources.require. - """ - result = [] - for p in sys_path: - # pkg_resources does not define a specific order for egg-link files - # using os.listdir to enumerate them, we're sorting them to have - # reproducible tests. - for egg_link in sorted(glob.glob(os.path.join(p, '*.egg-link'))): - with open(egg_link) as fd: - for line in fd: - line = line.strip() - if line: - result.append(os.path.join(p, line)) - # pkg_resources package only interprets the first - # non-empty line in egg-link files. - break - result.append(p) - return result - - -def _get_venv_path_dirs(venv): - """Get sys.path for venv without starting up the interpreter.""" - venv = os.path.abspath(venv) - sitedir = _get_venv_sitepackages(venv) - sys_path = [] - addsitedir(sys_path, sitedir) - return sys_path - - -def _get_venv_sitepackages(venv): - if os.name == 'nt': - p = os.path.join(venv, 'lib', 'site-packages') - else: - p = os.path.join(venv, 'lib', 'python%d.%d' % sys.version_info[:2], - 'site-packages') - return p - - -def _execute_code(module_path, code): - c = "import os; from os.path import *; result=%s" - variables = {'__file__': module_path} - try: - exec_function(c % code, variables) - except Exception: - debug.warning('sys.path manipulation detected, but failed to evaluate.') - else: - try: - res = variables['result'] - if isinstance(res, str): - return [os.path.abspath(res)] - except KeyError: - pass - return [] - - -def _paths_from_assignment(evaluator, expr_stmt): - """ - Extracts the assigned strings from an assignment that looks as follows:: - - >>> sys.path[0:0] = ['module/path', 'another/module/path'] - - This function is in general pretty tolerant (and therefore 'buggy'). - However, it's not a big issue usually to add more paths to Jedi's sys_path, - because it will only affect Jedi in very random situations and by adding - more paths than necessary, it usually benefits the general user. - """ - for assignee, operator in zip(expr_stmt.children[::2], expr_stmt.children[1::2]): - try: - assert operator in ['=', '+='] - assert tree.is_node(assignee, 'power', 'atom_expr') and \ - len(assignee.children) > 1 - c = assignee.children - assert c[0].type == 'name' and c[0].value == 'sys' - trailer = c[1] - assert trailer.children[0] == '.' and trailer.children[1].value == 'path' - # TODO Essentially we're not checking details on sys.path - # manipulation. Both assigment of the sys.path and changing/adding - # parts of the sys.path are the same: They get added to the current - # sys.path. - """ - execution = c[2] - assert execution.children[0] == '[' - subscript = execution.children[1] - assert subscript.type == 'subscript' - assert ':' in subscript.children - """ - except AssertionError: - continue - - from jedi.evaluate.iterable import py__iter__ - from jedi.evaluate.precedence import is_string - types = evaluator.eval_element(expr_stmt) - for types in py__iter__(evaluator, types, expr_stmt): - for typ in types: - if is_string(typ): - yield typ.obj - - -def _paths_from_list_modifications(module_path, trailer1, trailer2): - """ extract the path from either "sys.path.append" or "sys.path.insert" """ - # Guarantee that both are trailers, the first one a name and the second one - # a function execution with at least one param. - if not (tree.is_node(trailer1, 'trailer') and trailer1.children[0] == '.' - and tree.is_node(trailer2, 'trailer') and trailer2.children[0] == '(' - and len(trailer2.children) == 3): - return [] - - name = trailer1.children[1].value - if name not in ['insert', 'append']: - return [] - arg = trailer2.children[1] - if name == 'insert' and len(arg.children) in (3, 4): # Possible trailing comma. - arg = arg.children[2] - return _execute_code(module_path, arg.get_code()) - - -def _check_module(evaluator, module): - """ - Detect sys.path modifications within module. - """ - def get_sys_path_powers(names): - for name in names: - power = name.parent.parent - if tree.is_node(power, 'power', 'atom_expr'): - c = power.children - if isinstance(c[0], tree.Name) and c[0].value == 'sys' \ - and tree.is_node(c[1], 'trailer'): - n = c[1].children[1] - if isinstance(n, tree.Name) and n.value == 'path': - yield name, power - - sys_path = list(evaluator.sys_path) # copy - try: - possible_names = module.used_names['path'] - except KeyError: - # module.used_names is MergedNamesDict whose getitem never throws - # keyerror, this is superfluous. - pass - else: - for name, power in get_sys_path_powers(possible_names): - stmt = name.get_definition() - if len(power.children) >= 4: - sys_path.extend(_paths_from_list_modifications(module.path, *power.children[2:4])) - elif name.get_definition().type == 'expr_stmt': - sys_path.extend(_paths_from_assignment(evaluator, stmt)) - return sys_path - - -@memoize_default(evaluator_is_first_arg=True, default=[]) -def sys_path_with_modifications(evaluator, module): - if module.path is None: - # Support for modules without a path is bad, therefore return the - # normal path. - return list(evaluator.sys_path) - - curdir = os.path.abspath(os.curdir) - #TODO why do we need a chdir? - with common.ignored(OSError): - os.chdir(os.path.dirname(module.path)) - - buildout_script_paths = set() - - result = _check_module(evaluator, module) - result += _detect_django_path(module.path) - for buildout_script in _get_buildout_scripts(module.path): - for path in _get_paths_from_buildout_script(evaluator, buildout_script): - buildout_script_paths.add(path) - # cleanup, back to old directory - os.chdir(curdir) - return list(result) + list(buildout_script_paths) - - -def _get_paths_from_buildout_script(evaluator, buildout_script): - def load(buildout_script): - try: - with open(buildout_script, 'rb') as f: - source = common.source_to_unicode(f.read()) - except IOError: - debug.dbg('Error trying to read buildout_script: %s', buildout_script) - return - - p = ParserWithRecovery(evaluator.grammar, source, buildout_script) - save_parser(buildout_script, p) - return p.module - - cached = load_parser(buildout_script) - module = cached and cached.module or load(buildout_script) - if not module: - return - - for path in _check_module(evaluator, module): - yield path - - -def traverse_parents(path): - while True: - new = os.path.dirname(path) - if new == path: - return - path = new - yield path - - -def _get_parent_dir_with_file(path, filename): - for parent in traverse_parents(path): - if os.path.isfile(os.path.join(parent, filename)): - return parent - return None - - -def _detect_django_path(module_path): - """ Detects the path of the very well known Django library (if used) """ - result = [] - - for parent in traverse_parents(module_path): - with common.ignored(IOError): - with open(parent + os.path.sep + 'manage.py'): - debug.dbg('Found django path: %s', module_path) - result.append(parent) - return result - - -def _get_buildout_scripts(module_path): - """ - if there is a 'buildout.cfg' file in one of the parent directories of the - given module it will return a list of all files in the buildout bin - directory that look like python files. - - :param module_path: absolute path to the module. - :type module_path: str - """ - project_root = _get_parent_dir_with_file(module_path, 'buildout.cfg') - if not project_root: - return [] - bin_path = os.path.join(project_root, 'bin') - if not os.path.exists(bin_path): - return [] - extra_module_paths = [] - for filename in os.listdir(bin_path): - try: - filepath = os.path.join(bin_path, filename) - with open(filepath, 'r') as f: - firstline = f.readline() - if firstline.startswith('#!') and 'python' in firstline: - extra_module_paths.append(filepath) - except (UnicodeDecodeError, IOError) as e: - # Probably a binary file; permission error or race cond. because file got deleted - # ignore - debug.warning(unicode(e)) - continue - return extra_module_paths diff --git a/pythonFiles/preview/jedi/parser/__init__.py b/pythonFiles/preview/jedi/parser/__init__.py deleted file mode 100644 index 22f386050ef5..000000000000 --- a/pythonFiles/preview/jedi/parser/__init__.py +++ /dev/null @@ -1,409 +0,0 @@ -""" -The ``Parser`` tries to convert the available Python code in an easy to read -format, something like an abstract syntax tree. The classes who represent this -tree, are sitting in the :mod:`jedi.parser.tree` module. - -The Python module ``tokenize`` is a very important part in the ``Parser``, -because it splits the code into different words (tokens). Sometimes it looks a -bit messy. Sorry for that! You might ask now: "Why didn't you use the ``ast`` -module for this? Well, ``ast`` does a very good job understanding proper Python -code, but fails to work as soon as there's a single line of broken code. - -There's one important optimization that needs to be known: Statements are not -being parsed completely. ``Statement`` is just a representation of the tokens -within the statement. This lowers memory usage and cpu time and reduces the -complexity of the ``Parser`` (there's another parser sitting inside -``Statement``, which produces ``Array`` and ``Call``). -""" -import os -import re - -from jedi.parser import tree as pt -from jedi.parser import tokenize -from jedi.parser.token import (DEDENT, INDENT, ENDMARKER, NEWLINE, NUMBER, - STRING) -from jedi.parser.pgen2.pgen import generate_grammar -from jedi.parser.pgen2.parse import PgenParser - -OPERATOR_KEYWORDS = 'and', 'for', 'if', 'else', 'in', 'is', 'lambda', 'not', 'or' -# Not used yet. In the future I intend to add something like KeywordStatement -STATEMENT_KEYWORDS = 'assert', 'del', 'global', 'nonlocal', 'raise', \ - 'return', 'yield', 'pass', 'continue', 'break' - - -_loaded_grammars = {} - - -class ParseError(Exception): - """ - Signals you that the code you fed the Parser was not correct Python code. - """ - - -def load_grammar(version='3.4'): - # For now we only support two different Python syntax versions: The latest - # Python 3 and Python 2. This may change. - if version in ('3.2', '3.3'): - version = '3.4' - elif version == '2.6': - version = '2.7' - - file = 'grammar' + version + '.txt' - - global _loaded_grammars - path = os.path.join(os.path.dirname(__file__), file) - try: - return _loaded_grammars[path] - except KeyError: - return _loaded_grammars.setdefault(path, generate_grammar(path)) - - -class ParserSyntaxError(object): - def __init__(self, message, position): - self.message = message - self.position = position - - -class Parser(object): - AST_MAPPING = { - 'expr_stmt': pt.ExprStmt, - 'classdef': pt.Class, - 'funcdef': pt.Function, - 'file_input': pt.Module, - 'import_name': pt.ImportName, - 'import_from': pt.ImportFrom, - 'break_stmt': pt.KeywordStatement, - 'continue_stmt': pt.KeywordStatement, - 'return_stmt': pt.ReturnStmt, - 'raise_stmt': pt.KeywordStatement, - 'yield_expr': pt.YieldExpr, - 'del_stmt': pt.KeywordStatement, - 'pass_stmt': pt.KeywordStatement, - 'global_stmt': pt.GlobalStmt, - 'nonlocal_stmt': pt.KeywordStatement, - 'print_stmt': pt.KeywordStatement, - 'assert_stmt': pt.AssertStmt, - 'if_stmt': pt.IfStmt, - 'with_stmt': pt.WithStmt, - 'for_stmt': pt.ForStmt, - 'while_stmt': pt.WhileStmt, - 'try_stmt': pt.TryStmt, - 'comp_for': pt.CompFor, - 'decorator': pt.Decorator, - 'lambdef': pt.Lambda, - 'old_lambdef': pt.Lambda, - 'lambdef_nocond': pt.Lambda, - } - - def __init__(self, grammar, source, start_symbol='file_input', - tokenizer=None, start_parsing=True): - # Todo Remove start_parsing (with False) - - self._used_names = {} - self._scope_names_stack = [{}] - self._last_failed_start_pos = (0, 0) - self._global_names = [] - - # For the fast parser. - self.position_modifier = pt.PositionModifier() - - self._added_newline = False - # The Python grammar needs a newline at the end of each statement. - if not source.endswith('\n') and start_symbol == 'file_input': - source += '\n' - self._added_newline = True - - self.source = source - self._start_symbol = start_symbol - self._grammar = grammar - - self._parsed = None - - if start_parsing: - if tokenizer is None: - tokenizer = tokenize.source_tokens(source, use_exact_op_types=True) - self.parse(tokenizer) - - def parse(self, tokenizer): - if self._parsed is not None: - return self._parsed - - start_number = self._grammar.symbol2number[self._start_symbol] - pgen_parser = PgenParser( - self._grammar, self.convert_node, self.convert_leaf, - self.error_recovery, start_number - ) - - try: - self._parsed = pgen_parser.parse(tokenizer) - finally: - self.stack = pgen_parser.stack - - if self._start_symbol == 'file_input' != self._parsed.type: - # If there's only one statement, we get back a non-module. That's - # not what we want, we want a module, so we add it here: - self._parsed = self.convert_node(self._grammar, - self._grammar.symbol2number['file_input'], - [self._parsed]) - - if self._added_newline: - self.remove_last_newline() - - def get_parsed_node(self): - # TODO rename to get_root_node - return self._parsed - - def error_recovery(self, grammar, stack, arcs, typ, value, start_pos, prefix, - add_token_callback): - raise ParseError - - def convert_node(self, grammar, type, children): - """ - Convert raw node information to a Node instance. - - This is passed to the parser driver which calls it whenever a reduction of a - grammar rule produces a new complete node, so that the tree is build - strictly bottom-up. - """ - symbol = grammar.number2symbol[type] - try: - new_node = Parser.AST_MAPPING[symbol](children) - except KeyError: - new_node = pt.Node(symbol, children) - - # We need to check raw_node always, because the same node can be - # returned by convert multiple times. - if symbol == 'global_stmt': - self._global_names += new_node.get_global_names() - elif isinstance(new_node, pt.Lambda): - new_node.names_dict = self._scope_names_stack.pop() - elif isinstance(new_node, (pt.ClassOrFunc, pt.Module)) \ - and symbol in ('funcdef', 'classdef', 'file_input'): - # scope_name_stack handling - scope_names = self._scope_names_stack.pop() - if isinstance(new_node, pt.ClassOrFunc): - n = new_node.name - scope_names[n.value].remove(n) - # Set the func name of the current node - arr = self._scope_names_stack[-1].setdefault(n.value, []) - arr.append(n) - new_node.names_dict = scope_names - elif isinstance(new_node, pt.CompFor): - # The name definitions of comprehenions shouldn't be part of the - # current scope. They are part of the comprehension scope. - for n in new_node.get_defined_names(): - self._scope_names_stack[-1][n.value].remove(n) - return new_node - - def convert_leaf(self, grammar, type, value, prefix, start_pos): - # print('leaf', repr(value), token.tok_name[type]) - if type == tokenize.NAME: - if value in grammar.keywords: - if value in ('def', 'class', 'lambda'): - self._scope_names_stack.append({}) - - return pt.Keyword(self.position_modifier, value, start_pos, prefix) - else: - name = pt.Name(self.position_modifier, value, start_pos, prefix) - # Keep a listing of all used names - arr = self._used_names.setdefault(name.value, []) - arr.append(name) - arr = self._scope_names_stack[-1].setdefault(name.value, []) - arr.append(name) - return name - elif type == STRING: - return pt.String(self.position_modifier, value, start_pos, prefix) - elif type == NUMBER: - return pt.Number(self.position_modifier, value, start_pos, prefix) - elif type == NEWLINE: - return pt.Newline(self.position_modifier, value, start_pos, prefix) - elif type == INDENT: - return pt.Indent(self.position_modifier, value, start_pos, prefix) - elif type == DEDENT: - return pt.Dedent(self.position_modifier, value, start_pos, prefix) - elif type == ENDMARKER: - return pt.EndMarker(self.position_modifier, value, start_pos, prefix) - else: - return pt.Operator(self.position_modifier, value, start_pos, prefix) - - def remove_last_newline(self): - """ - In all of this we need to work with _start_pos, because if we worked - with start_pos, we would need to check the position_modifier as well - (which is accounted for in the start_pos property). - """ - endmarker = self._parsed.children[-1] - # The newline is either in the endmarker as a prefix or the previous - # leaf as a newline token. - prefix = endmarker.prefix - if prefix.endswith('\n'): - endmarker.prefix = prefix = prefix[:-1] - last_end = 0 - if '\n' not in prefix: - # Basically if the last line doesn't end with a newline. we - # have to add the previous line's end_position. - try: - last_end = endmarker.get_previous_leaf().end_pos[1] - except IndexError: - pass - last_line = re.sub('.*\n', '', prefix) - endmarker._start_pos = endmarker._start_pos[0] - 1, last_end + len(last_line) - else: - try: - newline = endmarker.get_previous_leaf() - except IndexError: - return # This means that the parser is empty. - while True: - if newline.value == '': - # Must be a DEDENT, just continue. - try: - newline = newline.get_previous_leaf() - except IndexError: - # If there's a statement that fails to be parsed, there - # will be no previous leaf. So just ignore it. - break - elif newline.value != '\n': - # TODO REMOVE, error recovery was simplified. - # This may happen if error correction strikes and removes - # a whole statement including '\n'. - break - else: - newline.value = '' - if self._last_failed_start_pos > newline._start_pos: - # It may be the case that there was a syntax error in a - # function. In that case error correction removes the - # right newline. So we use the previously assigned - # _last_failed_start_pos variable to account for that. - endmarker._start_pos = self._last_failed_start_pos - else: - endmarker._start_pos = newline._start_pos - break - - -class ParserWithRecovery(Parser): - """ - This class is used to parse a Python file, it then divides them into a - class structure of different scopes. - - :param grammar: The grammar object of pgen2. Loaded by load_grammar. - :param source: The codebase for the parser. Must be unicode. - :param module_path: The path of the module in the file system, may be None. - :type module_path: str - """ - def __init__(self, grammar, source, module_path=None, tokenizer=None, - start_parsing=True): - self.syntax_errors = [] - - self._omit_dedent_list = [] - self._indent_counter = 0 - - # TODO do print absolute import detection here. - # try: - # del python_grammar_no_print_statement.keywords["print"] - # except KeyError: - # pass # Doesn't exist in the Python 3 grammar. - - # if self.options["print_function"]: - # python_grammar = pygram.python_grammar_no_print_statement - # else: - super(ParserWithRecovery, self).__init__( - grammar, source, - tokenizer=tokenizer, - start_parsing=start_parsing - ) - if start_parsing: - self.module = self._parsed - self.module.used_names = self._used_names - self.module.path = module_path - self.module.global_names = self._global_names - - def parse(self, tokenizer): - return super(ParserWithRecovery, self).parse(self._tokenize(self._tokenize(tokenizer))) - - def error_recovery(self, grammar, stack, arcs, typ, value, start_pos, prefix, - add_token_callback): - """ - This parser is written in a dynamic way, meaning that this parser - allows using different grammars (even non-Python). However, error - recovery is purely written for Python. - """ - def current_suite(stack): - # For now just discard everything that is not a suite or - # file_input, if we detect an error. - for index, (dfa, state, (type_, nodes)) in reversed(list(enumerate(stack))): - # `suite` can sometimes be only simple_stmt, not stmt. - symbol = grammar.number2symbol[type_] - if symbol == 'file_input': - break - elif symbol == 'suite' and len(nodes) > 1: - # suites without an indent in them get discarded. - break - elif symbol == 'simple_stmt' and len(nodes) > 1: - # simple_stmt can just be turned into a Node, if there are - # enough statements. Ignore the rest after that. - break - return index, symbol, nodes - - index, symbol, nodes = current_suite(stack) - if symbol == 'simple_stmt': - index -= 2 - (_, _, (type_, suite_nodes)) = stack[index] - symbol = grammar.number2symbol[type_] - suite_nodes.append(pt.Node(symbol, list(nodes))) - # Remove - nodes[:] = [] - nodes = suite_nodes - stack[index] - - # print('err', token.tok_name[typ], repr(value), start_pos, len(stack), index) - if self._stack_removal(grammar, stack, arcs, index + 1, value, start_pos): - add_token_callback(typ, value, start_pos, prefix) - else: - if typ == INDENT: - # For every deleted INDENT we have to delete a DEDENT as well. - # Otherwise the parser will get into trouble and DEDENT too early. - self._omit_dedent_list.append(self._indent_counter) - else: - error_leaf = pt.ErrorLeaf(self.position_modifier, typ, value, start_pos, prefix) - stack[-1][2][1].append(error_leaf) - - def _stack_removal(self, grammar, stack, arcs, start_index, value, start_pos): - failed_stack = [] - found = False - all_nodes = [] - for dfa, state, (typ, nodes) in stack[start_index:]: - if nodes: - found = True - if found: - symbol = grammar.number2symbol[typ] - failed_stack.append((symbol, nodes)) - all_nodes += nodes - if nodes and nodes[0] in ('def', 'class', 'lambda'): - self._scope_names_stack.pop() - if failed_stack: - stack[start_index - 1][2][1].append(pt.ErrorNode(all_nodes)) - - self._last_failed_start_pos = start_pos - - stack[start_index:] = [] - return failed_stack - - def _tokenize(self, tokenizer): - for typ, value, start_pos, prefix in tokenizer: - # print(tokenize.tok_name[typ], repr(value), start_pos, repr(prefix)) - if typ == DEDENT: - # We need to count indents, because if we just omit any DEDENT, - # we might omit them in the wrong place. - o = self._omit_dedent_list - if o and o[-1] == self._indent_counter: - o.pop() - continue - - self._indent_counter -= 1 - elif typ == INDENT: - self._indent_counter += 1 - - yield typ, value, start_pos, prefix - - def __repr__(self): - return "<%s: %s>" % (type(self).__name__, self.module) diff --git a/pythonFiles/preview/jedi/parser/fast.py b/pythonFiles/preview/jedi/parser/fast.py deleted file mode 100644 index 9c471fd44336..000000000000 --- a/pythonFiles/preview/jedi/parser/fast.py +++ /dev/null @@ -1,621 +0,0 @@ -""" -Basically a parser that is faster, because it tries to parse only parts and if -anything changes, it only reparses the changed parts. But because it's not -finished (and still not working as I want), I won't document it any further. -""" -import re -from itertools import chain - -from jedi._compatibility import use_metaclass -from jedi import settings -from jedi.parser import ParserWithRecovery -from jedi.parser import tree -from jedi.parser.utils import underscore_memoization, parser_cache -from jedi import debug -from jedi.parser.tokenize import (source_tokens, NEWLINE, - ENDMARKER, INDENT, DEDENT) - -FLOWS = 'if', 'else', 'elif', 'while', 'with', 'try', 'except', 'finally', 'for' - - -class FastModule(tree.Module): - type = 'file_input' - - def __init__(self, module_path): - super(FastModule, self).__init__([]) - self.modules = [] - self.reset_caches() - self.names_dict = {} - self.path = module_path - - def reset_caches(self): - self.modules = [] - try: - del self._used_names # Remove the used names cache. - except AttributeError: - pass # It was never used. - - @property - @underscore_memoization - def used_names(self): - return MergedNamesDict([m.used_names for m in self.modules]) - - @property - def global_names(self): - return [name for m in self.modules for name in m.global_names] - - @property - def error_statements(self): - return [e for m in self.modules for e in m.error_statements] - - def __repr__(self): - return "" % (type(self).__name__, self.name, - self.start_pos[0], self.end_pos[0]) - - # To avoid issues with with the `parser.ParserWithRecovery`, we need - # setters that do nothing, because if pickle comes along and sets those - # values. - @global_names.setter - def global_names(self, value): - pass - - @error_statements.setter - def error_statements(self, value): - pass - - @used_names.setter - def used_names(self, value): - pass - - -class MergedNamesDict(object): - def __init__(self, dicts): - self.dicts = dicts - - def __iter__(self): - return iter(set(key for dct in self.dicts for key in dct)) - - def __getitem__(self, value): - return list(chain.from_iterable(dct.get(value, []) for dct in self.dicts)) - - def items(self): - dct = {} - for d in self.dicts: - for key, values in d.items(): - try: - dct_values = dct[key] - dct_values += values - except KeyError: - dct[key] = list(values) - return dct.items() - - def values(self): - lst = [] - for dct in self.dicts: - lst += dct.values() - return lst - - -class CachedFastParser(type): - """ This is a metaclass for caching `FastParser`. """ - def __call__(self, grammar, source, module_path=None): - if not settings.fast_parser: - return ParserWithRecovery(grammar, source, module_path) - - pi = parser_cache.get(module_path, None) - if pi is None or isinstance(pi.parser, ParserWithRecovery): - p = super(CachedFastParser, self).__call__(grammar, source, module_path) - else: - p = pi.parser # pi is a `cache.ParserCacheItem` - p.update(source) - return p - - -class ParserNode(object): - def __init__(self, fast_module, parser, source): - self._fast_module = fast_module - self.parent = None - self._node_children = [] - - self.source = source - self.hash = hash(source) - self.parser = parser - if source: - self._end_pos = parser.module.end_pos - else: - self._end_pos = 1, 0 - - try: - # With fast_parser we have either 1 subscope or only statements. - self._content_scope = parser.module.subscopes[0] - # A parsed node's content will be in the first indent, because - # everything that's parsed is within this subscope. - self._is_class_or_def = True - except IndexError: - self._content_scope = parser.module - self._is_class_or_def = False - else: - self._rewrite_last_newline() - - # We need to be able to reset the original children of a parser. - self._old_children = list(self._content_scope.children) - - def is_root_node(self): - return self.parent is None - - def _rewrite_last_newline(self): - """ - The ENDMARKER can contain a newline in the prefix. However this prefix - really belongs to the function - respectively to the next function or - parser node. If we don't rewrite that newline, we end up with a newline - in the wrong position, i.d. at the end of the file instead of in the - middle. - """ - c = self._content_scope.children - if tree.is_node(c[-1], 'suite'): # In a simple_stmt there's no DEDENT. - end_marker = self.parser.module.children[-1] - # Set the DEDENT prefix instead of the ENDMARKER. - c[-1].children[-1].prefix = end_marker.prefix - end_marker.prefix = '' - - def __repr__(self): - module = self.parser.module - try: - return '<%s: %s-%s>' % (type(self).__name__, module.start_pos, module.end_pos) - except IndexError: - # There's no module yet. - return '<%s: empty>' % type(self).__name__ - - @property - def end_pos(self): - return self._end_pos[0] + self.parser.position_modifier.line, self._end_pos[1] - - def reset_node(self): - """ - Removes changes that were applied in this class. - """ - self._node_children = [] - scope = self._content_scope - scope.children = list(self._old_children) - try: - # This works if it's a MergedNamesDict. - # We are correcting it, because the MergedNamesDicts are artificial - # and can change after closing a node. - scope.names_dict = scope.names_dict.dicts[0] - except AttributeError: - pass - - def close(self): - """ - Closes the current parser node. This means that after this no further - nodes should be added anymore. - """ - # We only need to replace the dict if multiple dictionaries are used: - if self._node_children: - dcts = [n.parser.module.names_dict for n in self._node_children] - # Need to insert the own node as well. - dcts.insert(0, self._content_scope.names_dict) - self._content_scope.names_dict = MergedNamesDict(dcts) - endmarker = self.parser.get_parsed_node().children[-1] - assert endmarker.type == 'endmarker' - last_parser = self._node_children[-1].parser - endmarker.start_pos = last_parser.get_parsed_node().end_pos - - @property - def _indent(self): - if self.is_root_node(): - return 0 - - return self.parser.module.children[0].start_pos[1] - - def add_node(self, node, start_line, indent): - """ - Adding a node means adding a node that was either just parsed or one - that can be reused. - """ - # Content that is not a subscope can never be part of the current node, - # because it's basically a sister node, that sits next to it and not - # within it. - if (self._indent >= indent or not self._is_class_or_def) and \ - not self.is_root_node(): - self.close() - return self.parent.add_node(node, start_line, indent) - - # Changing the line offsets is very important, because if they don't - # fit, all the start_pos values will be wrong. - m = node.parser.module - node.parser.position_modifier.line = start_line - 1 - self._fast_module.modules.append(m) - node.parent = self - - self._node_children.append(node) - - # Insert parser objects into current structure. We only need to set the - # parents and children in a good way. - scope = self._content_scope - for child in m.children: - child.parent = scope - scope.children.append(child) - - return node - - def all_sub_nodes(self): - """ - Returns all nodes including nested ones. - """ - for n in self._node_children: - yield n - for y in n.all_sub_nodes(): - yield y - - @underscore_memoization # Should only happen once! - def remove_last_newline(self): - self.parser.remove_last_newline() - - -class FastParser(use_metaclass(CachedFastParser)): - _FLOWS_NEED_SPACE = 'if', 'elif', 'while', 'with', 'except', 'for' - _FLOWS_NEED_COLON = 'else', 'try', 'except', 'finally' - _keyword_re = re.compile('^[ \t]*(def |class |@|(?:%s)|(?:%s)\s*:)' - % ('|'.join(_FLOWS_NEED_SPACE), - '|'.join(_FLOWS_NEED_COLON))) - - def __init__(self, grammar, source, module_path=None): - # set values like `tree.Module`. - self._grammar = grammar - self.module_path = module_path - self._reset_caches() - self.update(source) - - def _reset_caches(self): - self.module = FastModule(self.module_path) - self.root_node = self.current_node = ParserNode(self.module, self, '') - - def get_parsed_node(self): - return self.module - - def update(self, source): - # Variables for testing purposes: It is important that the number of - # parsers used can be minimized. With these variables we can test - # against that. - self.number_parsers_used = 0 - self.number_of_splits = 0 - self.number_of_misses = 0 - self.module.reset_caches() - self.source = source - try: - self._parse(source) - except: - # FastParser is cached, be careful with exceptions. - self._reset_caches() - raise - - def _split_parts(self, source): - """ - Split the source code into different parts. This makes it possible to - parse each part seperately and therefore cache parts of the file and - not everything. - """ - def gen_part(): - text = ''.join(current_lines) - del current_lines[:] - self.number_of_splits += 1 - return text - - def just_newlines(current_lines): - for line in current_lines: - line = line.lstrip('\t \n\r') - if line and line[0] != '#': - return False - return True - - # Split only new lines. Distinction between \r\n is the tokenizer's - # job. - # It seems like there's no problem with form feed characters here, - # because we're not counting lines. - self._lines = source.splitlines(True) - current_lines = [] - is_decorator = False - # Use -1, because that indent is always smaller than any other. - indent_list = [-1, 0] - new_indent = False - parentheses_level = 0 - flow_indent = None - previous_line = None - # All things within flows are simply being ignored. - for i, l in enumerate(self._lines): - # Handle backslash newline escaping. - if l.endswith('\\\n') or l.endswith('\\\r\n'): - if previous_line is not None: - previous_line += l - else: - previous_line = l - continue - if previous_line is not None: - l = previous_line + l - previous_line = None - - # check for dedents - s = l.lstrip('\t \n\r') - indent = len(l) - len(s) - if not s or s[0] == '#': - current_lines.append(l) # Just ignore comments and blank lines - continue - - if new_indent and not parentheses_level: - if indent > indent_list[-2]: - # Set the actual indent, not just the random old indent + 1. - indent_list[-1] = indent - new_indent = False - - while indent < indent_list[-1]: # -> dedent - indent_list.pop() - # This automatically resets the flow_indent if there was a - # dedent or a flow just on one line (with one simple_stmt). - new_indent = False - if flow_indent is None and current_lines and not parentheses_level: - yield gen_part() - flow_indent = None - - # Check lines for functions/classes and split the code there. - if flow_indent is None: - m = self._keyword_re.match(l) - if m: - # Strip whitespace and colon from flows as a check. - if m.group(1).strip(' \t\r\n:') in FLOWS: - if not parentheses_level: - flow_indent = indent - else: - if not is_decorator and not just_newlines(current_lines): - yield gen_part() - is_decorator = '@' == m.group(1) - if not is_decorator: - parentheses_level = 0 - # The new indent needs to be higher - indent_list.append(indent + 1) - new_indent = True - elif is_decorator: - is_decorator = False - - parentheses_level = \ - max(0, (l.count('(') + l.count('[') + l.count('{') - - l.count(')') - l.count(']') - l.count('}'))) - - current_lines.append(l) - - if previous_line is not None: - current_lines.append(previous_line) - if current_lines: - yield gen_part() - - def _parse(self, source): - """ :type source: str """ - added_newline = False - if not source or source[-1] != '\n': - # To be compatible with Pythons grammar, we need a newline at the - # end. The parser would handle it, but since the fast parser abuses - # the normal parser in various ways, we need to care for this - # ourselves. - source += '\n' - added_newline = True - - next_code_part_end_line = code_part_end_line = 1 - start = 0 - nodes = list(self.root_node.all_sub_nodes()) - # Now we can reset the node, because we have all the old nodes. - self.root_node.reset_node() - self.current_node = self.root_node - last_end_line = 1 - - for code_part in self._split_parts(source): - next_code_part_end_line += code_part.count('\n') - # If the last code part parsed isn't equal to the current end_pos, - # we know that the parser went further (`def` start in a - # docstring). So just parse the next part. - if code_part_end_line == last_end_line: - self._parse_part(code_part, source[start:], code_part_end_line, nodes) - else: - self.number_of_misses += 1 - # Means that some lines where not fully parsed. Parse it now. - # This is a very rare case. Should only happens with very - # strange code bits. - while last_end_line < next_code_part_end_line: - code_part_end_line = last_end_line - # We could calculate the src in a more complicated way to - # make caching here possible as well. However, this is - # complicated and error-prone. Since this is not very often - # called - just ignore it. - src = ''.join(self._lines[code_part_end_line - 1:]) - self._parse_part(code_part, src, code_part_end_line, nodes) - last_end_line = self.current_node.end_pos[0] - debug.dbg("While parsing %s, starting with line %s wasn't included in split.", - self.module_path, code_part_end_line) - #assert code_part_end_line > last_end_line - # This means that the parser parsed faster than the last given - # `code_part`. - debug.dbg('While parsing %s, line %s slowed down the fast parser.', - self.module_path, code_part_end_line) - - code_part_end_line = next_code_part_end_line - start += len(code_part) - - last_end_line = self.current_node.end_pos[0] - - if added_newline: - self.current_node.remove_last_newline() - - # Now that the for loop is finished, we still want to close all nodes. - node = self.current_node - while node is not None: - node.close() - node = node.parent - - debug.dbg('Parsed %s, with %s parsers in %s splits.' - % (self.module_path, self.number_parsers_used, - self.number_of_splits)) - - def _parse_part(self, source, parser_code, code_part_end_line, nodes): - """ - Side effect: Alters the list of nodes. - """ - h = hash(source) - for index, node in enumerate(nodes): - if node.hash == h and node.source == source: - node.reset_node() - nodes.remove(node) - parser_code = source - break - else: - tokenizer = FastTokenizer(parser_code) - self.number_parsers_used += 1 - p = ParserWithRecovery(self._grammar, parser_code, self.module_path, tokenizer=tokenizer) - - end = code_part_end_line - 1 + p.module.end_pos[0] - used_lines = self._lines[code_part_end_line - 1:end - 1] - code_part_actually_used = ''.join(used_lines) - - node = ParserNode(self.module, p, code_part_actually_used) - - indent = len(parser_code) - len(parser_code.lstrip('\t ')) - - self.current_node.add_node(node, code_part_end_line, indent) - self.current_node = node - - -class FastTokenizer(object): - """ - Breaks when certain conditions are met, i.e. a new function or class opens. - """ - def __init__(self, source): - self.source = source - self._gen = source_tokens(source, use_exact_op_types=True) - self._closed = False - - # fast parser options - self.current = self.previous = NEWLINE, '', (0, 0) - self._in_flow = False - self._is_decorator = False - self._first_stmt = True - self._parentheses_level = 0 - self._indent_counter = 0 - self._flow_indent_counter = 0 - self._returned_endmarker = False - self._expect_indent = False - - def __iter__(self): - return self - - def next(self): - """ Python 2 Compatibility """ - return self.__next__() - - def __next__(self): - if self._closed: - return self._finish_dedents() - - typ, value, start_pos, prefix = current = next(self._gen) - if typ == ENDMARKER: - self._closed = True - self._returned_endmarker = True - return current - - self.previous = self.current - self.current = current - - if typ == INDENT: - self._indent_counter += 1 - if not self._expect_indent and not self._first_stmt and not self._in_flow: - # This does not mean that there is an actual flow, it means - # that the INDENT is syntactically wrong. - self._flow_indent_counter = self._indent_counter - 1 - self._in_flow = True - self._expect_indent = False - elif typ == DEDENT: - self._indent_counter -= 1 - if self._in_flow: - if self._indent_counter == self._flow_indent_counter: - self._in_flow = False - else: - self._closed = True - return current - - previous_type = self.previous[0] - if value in ('def', 'class') and self._parentheses_level: - # Account for the fact that an open parentheses before a function - # will reset the parentheses counter, but new lines before will - # still be ignored. So check the prefix. - - # TODO what about flow parentheses counter resets in the tokenizer? - self._parentheses_level = 0 - # We need to simulate a newline before the indent, because the - # open parentheses ignored them. - if re.search('\n\s*', prefix): - previous_type = NEWLINE - - # Parentheses ignore the indentation rules. The other three stand for - # new lines. - if previous_type in (NEWLINE, INDENT, DEDENT) \ - and not self._parentheses_level and typ not in (INDENT, DEDENT): - if not self._in_flow: - if value in FLOWS: - self._flow_indent_counter = self._indent_counter - self._first_stmt = False - elif value in ('def', 'class', '@'): - # The values here are exactly the same check as in - # _split_parts, but this time with tokenize and therefore - # precise. - if not self._first_stmt and not self._is_decorator: - return self._close() - - self._is_decorator = '@' == value - if not self._is_decorator: - self._first_stmt = False - self._expect_indent = True - elif self._expect_indent: - return self._close() - else: - self._first_stmt = False - - if value in '([{' and value: - self._parentheses_level += 1 - elif value in ')]}' and value: - # Ignore closing parentheses, because they are all - # irrelevant for the indentation. - self._parentheses_level = max(self._parentheses_level - 1, 0) - return current - - def _close(self): - if self._first_stmt: - # Continue like nothing has happened, because we want to enter - # the first class/function. - if self.current[1] != '@': - self._first_stmt = False - return self.current - else: - self._closed = True - return self._finish_dedents() - - def _finish_dedents(self): - if self._indent_counter: - self._indent_counter -= 1 - return DEDENT, '', self.current[2], '' - elif not self._returned_endmarker: - self._returned_endmarker = True - return ENDMARKER, '', self.current[2], self._get_prefix() - else: - raise StopIteration - - def _get_prefix(self): - """ - We're using the current prefix for the endmarker to not loose any - information. However we care about "lost" lines. The prefix of the - current line (indent) will always be included in the current line. - """ - cur = self.current - while cur[0] == DEDENT: - cur = next(self._gen) - prefix = cur[3] - - # \Z for the end of the string. $ is bugged, because it has the - # same behavior with or without re.MULTILINE. - return re.sub(r'[^\n]+\Z', '', prefix) diff --git a/pythonFiles/preview/jedi/parser/pgen2/pgen.py b/pythonFiles/preview/jedi/parser/pgen2/pgen.py deleted file mode 100644 index fa2742dd5dc4..000000000000 --- a/pythonFiles/preview/jedi/parser/pgen2/pgen.py +++ /dev/null @@ -1,394 +0,0 @@ -# Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -# Modifications: -# Copyright 2014 David Halter. Integration into Jedi. -# Modifications are dual-licensed: MIT and PSF. - -# Pgen imports -from . import grammar -from jedi.parser import token -from jedi.parser import tokenize - - -class ParserGenerator(object): - def __init__(self, filename, stream=None): - close_stream = None - if stream is None: - stream = open(filename) - close_stream = stream.close - self.filename = filename - self.stream = stream - self.generator = tokenize.generate_tokens(stream.readline) - self.gettoken() # Initialize lookahead - self.dfas, self.startsymbol = self.parse() - if close_stream is not None: - close_stream() - self.first = {} # map from symbol name to set of tokens - self.addfirstsets() - - def make_grammar(self): - c = grammar.Grammar() - names = list(self.dfas.keys()) - names.sort() - names.remove(self.startsymbol) - names.insert(0, self.startsymbol) - for name in names: - i = 256 + len(c.symbol2number) - c.symbol2number[name] = i - c.number2symbol[i] = name - for name in names: - dfa = self.dfas[name] - states = [] - for state in dfa: - arcs = [] - for label, next in state.arcs.items(): - arcs.append((self.make_label(c, label), dfa.index(next))) - if state.isfinal: - arcs.append((0, dfa.index(state))) - states.append(arcs) - c.states.append(states) - c.dfas[c.symbol2number[name]] = (states, self.make_first(c, name)) - c.start = c.symbol2number[self.startsymbol] - return c - - def make_first(self, c, name): - rawfirst = self.first[name] - first = {} - for label in rawfirst: - ilabel = self.make_label(c, label) - ##assert ilabel not in first # XXX failed on <> ... != - first[ilabel] = 1 - return first - - def make_label(self, c, label): - # XXX Maybe this should be a method on a subclass of converter? - ilabel = len(c.labels) - if label[0].isalpha(): - # Either a symbol name or a named token - if label in c.symbol2number: - # A symbol name (a non-terminal) - if label in c.symbol2label: - return c.symbol2label[label] - else: - c.labels.append((c.symbol2number[label], None)) - c.symbol2label[label] = ilabel - return ilabel - else: - # A named token (NAME, NUMBER, STRING) - itoken = getattr(token, label, None) - assert isinstance(itoken, int), label - assert itoken in token.tok_name, label - if itoken in c.tokens: - return c.tokens[itoken] - else: - c.labels.append((itoken, None)) - c.tokens[itoken] = ilabel - return ilabel - else: - # Either a keyword or an operator - assert label[0] in ('"', "'"), label - value = eval(label) - if value[0].isalpha(): - # A keyword - if value in c.keywords: - return c.keywords[value] - else: - c.labels.append((token.NAME, value)) - c.keywords[value] = ilabel - return ilabel - else: - # An operator (any non-numeric token) - itoken = token.opmap[value] # Fails if unknown token - if itoken in c.tokens: - return c.tokens[itoken] - else: - c.labels.append((itoken, None)) - c.tokens[itoken] = ilabel - return ilabel - - def addfirstsets(self): - names = list(self.dfas.keys()) - names.sort() - for name in names: - if name not in self.first: - self.calcfirst(name) - #print name, self.first[name].keys() - - def calcfirst(self, name): - dfa = self.dfas[name] - self.first[name] = None # dummy to detect left recursion - state = dfa[0] - totalset = {} - overlapcheck = {} - for label, next in state.arcs.items(): - if label in self.dfas: - if label in self.first: - fset = self.first[label] - if fset is None: - raise ValueError("recursion for rule %r" % name) - else: - self.calcfirst(label) - fset = self.first[label] - totalset.update(fset) - overlapcheck[label] = fset - else: - totalset[label] = 1 - overlapcheck[label] = {label: 1} - inverse = {} - for label, itsfirst in overlapcheck.items(): - for symbol in itsfirst: - if symbol in inverse: - raise ValueError("rule %s is ambiguous; %s is in the" - " first sets of %s as well as %s" % - (name, symbol, label, inverse[symbol])) - inverse[symbol] = label - self.first[name] = totalset - - def parse(self): - dfas = {} - startsymbol = None - # MSTART: (NEWLINE | RULE)* ENDMARKER - while self.type != token.ENDMARKER: - while self.type == token.NEWLINE: - self.gettoken() - # RULE: NAME ':' RHS NEWLINE - name = self.expect(token.NAME) - self.expect(token.OP, ":") - a, z = self.parse_rhs() - self.expect(token.NEWLINE) - #self.dump_nfa(name, a, z) - dfa = self.make_dfa(a, z) - #self.dump_dfa(name, dfa) - # oldlen = len(dfa) - self.simplify_dfa(dfa) - # newlen = len(dfa) - dfas[name] = dfa - #print name, oldlen, newlen - if startsymbol is None: - startsymbol = name - return dfas, startsymbol - - def make_dfa(self, start, finish): - # To turn an NFA into a DFA, we define the states of the DFA - # to correspond to *sets* of states of the NFA. Then do some - # state reduction. Let's represent sets as dicts with 1 for - # values. - assert isinstance(start, NFAState) - assert isinstance(finish, NFAState) - - def closure(state): - base = {} - addclosure(state, base) - return base - - def addclosure(state, base): - assert isinstance(state, NFAState) - if state in base: - return - base[state] = 1 - for label, next in state.arcs: - if label is None: - addclosure(next, base) - - states = [DFAState(closure(start), finish)] - for state in states: # NB states grows while we're iterating - arcs = {} - for nfastate in state.nfaset: - for label, next in nfastate.arcs: - if label is not None: - addclosure(next, arcs.setdefault(label, {})) - for label, nfaset in arcs.items(): - for st in states: - if st.nfaset == nfaset: - break - else: - st = DFAState(nfaset, finish) - states.append(st) - state.addarc(st, label) - return states # List of DFAState instances; first one is start - - def dump_nfa(self, name, start, finish): - print("Dump of NFA for", name) - todo = [start] - for i, state in enumerate(todo): - print(" State", i, state is finish and "(final)" or "") - for label, next in state.arcs: - if next in todo: - j = todo.index(next) - else: - j = len(todo) - todo.append(next) - if label is None: - print(" -> %d" % j) - else: - print(" %s -> %d" % (label, j)) - - def dump_dfa(self, name, dfa): - print("Dump of DFA for", name) - for i, state in enumerate(dfa): - print(" State", i, state.isfinal and "(final)" or "") - for label, next in state.arcs.items(): - print(" %s -> %d" % (label, dfa.index(next))) - - def simplify_dfa(self, dfa): - # This is not theoretically optimal, but works well enough. - # Algorithm: repeatedly look for two states that have the same - # set of arcs (same labels pointing to the same nodes) and - # unify them, until things stop changing. - - # dfa is a list of DFAState instances - changes = True - while changes: - changes = False - for i, state_i in enumerate(dfa): - for j in range(i + 1, len(dfa)): - state_j = dfa[j] - if state_i == state_j: - #print " unify", i, j - del dfa[j] - for state in dfa: - state.unifystate(state_j, state_i) - changes = True - break - - def parse_rhs(self): - # RHS: ALT ('|' ALT)* - a, z = self.parse_alt() - if self.value != "|": - return a, z - else: - aa = NFAState() - zz = NFAState() - aa.addarc(a) - z.addarc(zz) - while self.value == "|": - self.gettoken() - a, z = self.parse_alt() - aa.addarc(a) - z.addarc(zz) - return aa, zz - - def parse_alt(self): - # ALT: ITEM+ - a, b = self.parse_item() - while (self.value in ("(", "[") or - self.type in (token.NAME, token.STRING)): - c, d = self.parse_item() - b.addarc(c) - b = d - return a, b - - def parse_item(self): - # ITEM: '[' RHS ']' | ATOM ['+' | '*'] - if self.value == "[": - self.gettoken() - a, z = self.parse_rhs() - self.expect(token.OP, "]") - a.addarc(z) - return a, z - else: - a, z = self.parse_atom() - value = self.value - if value not in ("+", "*"): - return a, z - self.gettoken() - z.addarc(a) - if value == "+": - return a, z - else: - return a, a - - def parse_atom(self): - # ATOM: '(' RHS ')' | NAME | STRING - if self.value == "(": - self.gettoken() - a, z = self.parse_rhs() - self.expect(token.OP, ")") - return a, z - elif self.type in (token.NAME, token.STRING): - a = NFAState() - z = NFAState() - a.addarc(z, self.value) - self.gettoken() - return a, z - else: - self.raise_error("expected (...) or NAME or STRING, got %s/%s", - self.type, self.value) - - def expect(self, type, value=None): - if self.type != type or (value is not None and self.value != value): - self.raise_error("expected %s/%s, got %s/%s", - type, value, self.type, self.value) - value = self.value - self.gettoken() - return value - - def gettoken(self): - tup = next(self.generator) - while tup[0] in (token.COMMENT, token.NL): - tup = next(self.generator) - self.type, self.value, self.begin, prefix = tup - #print tokenize.tok_name[self.type], repr(self.value) - - def raise_error(self, msg, *args): - if args: - try: - msg = msg % args - except: - msg = " ".join([msg] + list(map(str, args))) - line = open(self.filename).readlines()[self.begin[0]] - raise SyntaxError(msg, (self.filename, self.begin[0], - self.begin[1], line)) - - -class NFAState(object): - def __init__(self): - self.arcs = [] # list of (label, NFAState) pairs - - def addarc(self, next, label=None): - assert label is None or isinstance(label, str) - assert isinstance(next, NFAState) - self.arcs.append((label, next)) - - -class DFAState(object): - def __init__(self, nfaset, final): - assert isinstance(nfaset, dict) - assert isinstance(next(iter(nfaset)), NFAState) - assert isinstance(final, NFAState) - self.nfaset = nfaset - self.isfinal = final in nfaset - self.arcs = {} # map from label to DFAState - - def addarc(self, next, label): - assert isinstance(label, str) - assert label not in self.arcs - assert isinstance(next, DFAState) - self.arcs[label] = next - - def unifystate(self, old, new): - for label, next in self.arcs.items(): - if next is old: - self.arcs[label] = new - - def __eq__(self, other): - # Equality test -- ignore the nfaset instance variable - assert isinstance(other, DFAState) - if self.isfinal != other.isfinal: - return False - # Can't just return self.arcs == other.arcs, because that - # would invoke this method recursively, with cycles... - if len(self.arcs) != len(other.arcs): - return False - for label, next in self.arcs.items(): - if next is not other.arcs.get(label): - return False - return True - - __hash__ = None # For Py3 compatibility. - - -def generate_grammar(filename="Grammar.txt"): - p = ParserGenerator(filename) - return p.make_grammar() diff --git a/pythonFiles/preview/jedi/parser/tokenize.py b/pythonFiles/preview/jedi/parser/tokenize.py deleted file mode 100644 index 233c6d877398..000000000000 --- a/pythonFiles/preview/jedi/parser/tokenize.py +++ /dev/null @@ -1,329 +0,0 @@ -# -*- coding: utf-8 -*- -""" -This tokenizer has been copied from the ``tokenize.py`` standard library -tokenizer. The reason was simple: The standard library tokenizer fails -if the indentation is not right. The fast parser of jedi however requires -"wrong" indentation. - -Basically this is a stripped down version of the standard library module, so -you can read the documentation there. Additionally we included some speed and -memory optimizations here. -""" -from __future__ import absolute_import - -import string -import re -from collections import namedtuple -from io import StringIO - -from jedi.parser.token import (tok_name, N_TOKENS, ENDMARKER, STRING, NUMBER, opmap, - NAME, OP, ERRORTOKEN, NEWLINE, INDENT, DEDENT) -from jedi._compatibility import is_py3 - - -cookie_re = re.compile("coding[:=]\s*([-\w.]+)") - - -if is_py3: - # Python 3 has str.isidentifier() to check if a char is a valid identifier - is_identifier = str.isidentifier -else: - namechars = string.ascii_letters + '_' - is_identifier = lambda s: s in namechars - - -COMMENT = N_TOKENS -tok_name[COMMENT] = 'COMMENT' - - -def group(*choices): - return '(' + '|'.join(choices) + ')' - - -def maybe(*choices): - return group(*choices) + '?' - - -# Note: we use unicode matching for names ("\w") but ascii matching for -# number literals. -whitespace = r'[ \f\t]*' -comment = r'#[^\r\n]*' -name = r'\w+' - -hex_number = r'0[xX][0-9a-fA-F]+' -bin_number = r'0[bB][01]+' -if is_py3: - oct_number = r'0[oO][0-7]+' -else: - oct_number = '0[0-7]+' -dec_number = r'(?:0+|[1-9][0-9]*)' -int_number = group(hex_number, bin_number, oct_number, dec_number) -exponent = r'[eE][-+]?[0-9]+' -point_float = group(r'[0-9]+\.[0-9]*', r'\.[0-9]+') + maybe(exponent) -Expfloat = r'[0-9]+' + exponent -float_number = group(point_float, Expfloat) -imag_number = group(r'[0-9]+[jJ]', float_number + r'[jJ]') -number = group(imag_number, float_number, int_number) - -# Tail end of ' string. -single = r"[^'\\]*(?:\\.[^'\\]*)*'" -# Tail end of " string. -double = r'[^"\\]*(?:\\.[^"\\]*)*"' -# Tail end of ''' string. -single3 = r"[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''" -# Tail end of """ string. -double3 = r'[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""' -triple = group("[uUbB]?[rR]?'''", '[uUbB]?[rR]?"""') -# Single-line ' or " string. - -# Because of leftmost-then-longest match semantics, be sure to put the -# longest operators first (e.g., if = came before ==, == would get -# recognized as two instances of =). -operator = group(r"\*\*=?", r">>=?", r"<<=?", r"!=", - r"//=?", r"->", - r"[+\-*@/%&|^=<>]=?", - r"~") - -bracket = '[][(){}]' -special = group(r'\r?\n', r'\.\.\.', r'[:;.,@]') -funny = group(operator, bracket, special) - -# First (or only) line of ' or " string. -cont_str = group(r"[bBuU]?[rR]?'[^\n'\\]*(?:\\.[^\n'\\]*)*" + - group("'", r'\\\r?\n'), - r'[bBuU]?[rR]?"[^\n"\\]*(?:\\.[^\n"\\]*)*' + - group('"', r'\\\r?\n')) -pseudo_extras = group(r'\\\r?\n', comment, triple) -pseudo_token = group(whitespace) + \ - group(pseudo_extras, number, funny, cont_str, name) - - -def _compile(expr): - return re.compile(expr, re.UNICODE) - - -pseudoprog, single3prog, double3prog = map( - _compile, (pseudo_token, single3, double3)) - -endprogs = {"'": _compile(single), '"': _compile(double), - "'''": single3prog, '"""': double3prog, - "r'''": single3prog, 'r"""': double3prog, - "b'''": single3prog, 'b"""': double3prog, - "u'''": single3prog, 'u"""': double3prog, - "R'''": single3prog, 'R"""': double3prog, - "B'''": single3prog, 'B"""': double3prog, - "U'''": single3prog, 'U"""': double3prog, - "br'''": single3prog, 'br"""': double3prog, - "bR'''": single3prog, 'bR"""': double3prog, - "Br'''": single3prog, 'Br"""': double3prog, - "BR'''": single3prog, 'BR"""': double3prog, - "ur'''": single3prog, 'ur"""': double3prog, - "uR'''": single3prog, 'uR"""': double3prog, - "Ur'''": single3prog, 'Ur"""': double3prog, - "UR'''": single3prog, 'UR"""': double3prog, - 'r': None, 'R': None, 'b': None, 'B': None} - -triple_quoted = {} -for t in ("'''", '"""', - "r'''", 'r"""', "R'''", 'R"""', - "b'''", 'b"""', "B'''", 'B"""', - "u'''", 'u"""', "U'''", 'U"""', - "br'''", 'br"""', "Br'''", 'Br"""', - "bR'''", 'bR"""', "BR'''", 'BR"""', - "ur'''", 'ur"""', "Ur'''", 'Ur"""', - "uR'''", 'uR"""', "UR'''", 'UR"""'): - triple_quoted[t] = t -single_quoted = {} -for t in ("'", '"', - "r'", 'r"', "R'", 'R"', - "b'", 'b"', "B'", 'B"', - "u'", 'u"', "U'", 'U"', - "br'", 'br"', "Br'", 'Br"', - "bR'", 'bR"', "BR'", 'BR"', - "ur'", 'ur"', "Ur'", 'Ur"', - "uR'", 'uR"', "UR'", 'UR"'): - single_quoted[t] = t - -del _compile - -tabsize = 8 - -# TODO add with? -ALWAYS_BREAK_TOKENS = (';', 'import', 'class', 'def', 'try', 'except', - 'finally', 'while', 'return') - - -class TokenInfo(namedtuple('Token', ['type', 'string', 'start_pos', 'prefix'])): - def __repr__(self): - annotated_type = tok_name[self.type] - return ('TokenInfo(type=%s, string=%r, start=%r, prefix=%r)' % - self._replace(type=annotated_type)) - - @property - def exact_type(self): - if self.type == OP and self.string in opmap: - return opmap[self.string] - else: - return self.type - - -def source_tokens(source, use_exact_op_types=False): - """Generate tokens from a the source code (string).""" - source = source - readline = StringIO(source).readline - return generate_tokens(readline, use_exact_op_types) - - -def generate_tokens(readline, use_exact_op_types=False): - """ - A heavily modified Python standard library tokenizer. - - Additionally to the default information, yields also the prefix of each - token. This idea comes from lib2to3. The prefix contains all information - that is irrelevant for the parser like newlines in parentheses or comments. - """ - paren_level = 0 # count parentheses - indents = [0] - lnum = 0 - max = 0 - numchars = '0123456789' - contstr = '' - contline = None - # We start with a newline. This makes indent at the first position - # possible. It's not valid Python, but still better than an INDENT in the - # second line (and not in the first). This makes quite a few things in - # Jedi's fast parser possible. - new_line = True - prefix = '' # Should never be required, but here for safety - additional_prefix = '' - while True: # loop over lines in stream - line = readline() # readline returns empty when finished. See StringIO - if not line: - if contstr: - yield TokenInfo(ERRORTOKEN, contstr, contstr_start, prefix) - break - - lnum += 1 - pos, max = 0, len(line) - - if contstr: # continued string - endmatch = endprog.match(line) - if endmatch: - pos = endmatch.end(0) - yield TokenInfo(STRING, contstr + line[:pos], contstr_start, prefix) - contstr = '' - contline = None - else: - contstr = contstr + line - contline = contline + line - continue - - while pos < max: - pseudomatch = pseudoprog.match(line, pos) - if not pseudomatch: # scan for tokens - txt = line[pos] - if line[pos] in '"\'': - # If a literal starts but doesn't end the whole rest of the - # line is an error token. - txt = line[pos:] - yield TokenInfo(ERRORTOKEN, txt, (lnum, pos), prefix) - pos += 1 - continue - - prefix = additional_prefix + pseudomatch.group(1) - additional_prefix = '' - start, pos = pseudomatch.span(2) - spos = (lnum, start) - token, initial = line[start:pos], line[start] - - if new_line and initial not in '\r\n#': - new_line = False - if paren_level == 0: - i = 0 - while line[i] == '\f': - i += 1 - start -= 1 - if start > indents[-1]: - yield TokenInfo(INDENT, '', spos, '') - indents.append(start) - while start < indents[-1]: - yield TokenInfo(DEDENT, '', spos, '') - indents.pop() - - if (initial in numchars or # ordinary number - (initial == '.' and token != '.' and token != '...')): - yield TokenInfo(NUMBER, token, spos, prefix) - elif initial in '\r\n': - if not new_line and paren_level == 0: - yield TokenInfo(NEWLINE, token, spos, prefix) - else: - additional_prefix = prefix + token - new_line = True - elif initial == '#': # Comments - assert not token.endswith("\n") - additional_prefix = prefix + token - elif token in triple_quoted: - endprog = endprogs[token] - endmatch = endprog.match(line, pos) - if endmatch: # all on one line - pos = endmatch.end(0) - token = line[start:pos] - yield TokenInfo(STRING, token, spos, prefix) - else: - contstr_start = (lnum, start) # multiple lines - contstr = line[start:] - contline = line - break - elif initial in single_quoted or \ - token[:2] in single_quoted or \ - token[:3] in single_quoted: - if token[-1] == '\n': # continued string - contstr_start = lnum, start - endprog = (endprogs.get(initial) or endprogs.get(token[1]) - or endprogs.get(token[2])) - contstr = line[start:] - contline = line - break - else: # ordinary string - yield TokenInfo(STRING, token, spos, prefix) - elif is_identifier(initial): # ordinary name - if token in ALWAYS_BREAK_TOKENS: - paren_level = 0 - while True: - indent = indents.pop() - if indent > start: - yield TokenInfo(DEDENT, '', spos, '') - else: - indents.append(indent) - break - yield TokenInfo(NAME, token, spos, prefix) - elif initial == '\\' and line[start:] in ('\\\n', '\\\r\n'): # continued stmt - additional_prefix += prefix + line[start:] - break - else: - if token in '([{': - paren_level += 1 - elif token in ')]}': - paren_level -= 1 - - try: - # This check is needed in any case to check if it's a valid - # operator or just some random unicode character. - exact_type = opmap[token] - except KeyError: - exact_type = typ = ERRORTOKEN - if use_exact_op_types: - typ = exact_type - else: - typ = OP - yield TokenInfo(typ, token, spos, prefix) - - if new_line or additional_prefix[-1:] == '\n': - end_pos = lnum + 1, 0 - else: - end_pos = lnum, max - # As the last position we just take the maximally possible position. We - # remove -1 for the last new line. - for indent in indents[1:]: - yield TokenInfo(DEDENT, '', end_pos, '') - yield TokenInfo(ENDMARKER, '', end_pos, additional_prefix) diff --git a/pythonFiles/preview/jedi/parser/tree.py b/pythonFiles/preview/jedi/parser/tree.py deleted file mode 100644 index 0033a49cbfde..000000000000 --- a/pythonFiles/preview/jedi/parser/tree.py +++ /dev/null @@ -1,1663 +0,0 @@ -""" -If you know what an abstract syntax tree (AST) is, you'll see that this module -is pretty much that. The classes represent syntax elements like functions and -imports. - -This is the "business logic" part of the parser. There's a lot of logic here -that makes it easier for Jedi (and other libraries to deal with a Python syntax -tree. - -By using `get_code` on a module, you can get back the 1-to-1 representation of -the input given to the parser. This is important if you are using refactoring. - -The easiest way to play with this module is to use :class:`parsing.Parser`. -:attr:`parsing.Parser.module` holds an instance of :class:`Module`: - ->>> from jedi._compatibility import u ->>> from jedi.parser import ParserWithRecovery, load_grammar ->>> parser = ParserWithRecovery(load_grammar(), u('import os'), 'example.py') ->>> submodule = parser.module ->>> submodule - - -Any subclasses of :class:`Scope`, including :class:`Module` has an attribute -:attr:`imports `: - ->>> submodule.imports -[] - -See also :attr:`Scope.subscopes` and :attr:`Scope.statements`. - -For static analysis purposes there exists a method called -``nodes_to_execute`` on all nodes and leaves. It's documented in the static -anaylsis documentation. -""" -import os -import re -from inspect import cleandoc -from itertools import chain -import textwrap -import abc - -from jedi._compatibility import (Python3Method, encoding, is_py3, utf8_repr, - literal_eval, use_metaclass, unicode) -from jedi.parser import token -from jedi.parser.utils import underscore_memoization - - -def is_node(node, *symbol_names): - try: - type = node.type - except AttributeError: - return False - else: - return type in symbol_names - - -class PositionModifier(object): - """A start_pos modifier for the fast parser.""" - def __init__(self): - self.line = 0 - - -zero_position_modifier = PositionModifier() - - -class DocstringMixin(object): - __slots__ = () - - @property - def raw_doc(self): - """ Returns a cleaned version of the docstring token. """ - if isinstance(self, Module): - node = self.children[0] - elif isinstance(self, ClassOrFunc): - node = self.children[self.children.index(':') + 1] - if is_node(node, 'suite'): # Normally a suite - node = node.children[2] # -> NEWLINE INDENT stmt - else: # ExprStmt - simple_stmt = self.parent - c = simple_stmt.parent.children - index = c.index(simple_stmt) - if not index: - return '' - node = c[index - 1] - - if is_node(node, 'simple_stmt'): - node = node.children[0] - - if node.type == 'string': - # TODO We have to check next leaves until there are no new - # leaves anymore that might be part of the docstring. A - # docstring can also look like this: ``'foo' 'bar' - # Returns a literal cleaned version of the ``Token``. - cleaned = cleandoc(literal_eval(node.value)) - # Since we want the docstr output to be always unicode, just - # force it. - if is_py3 or isinstance(cleaned, unicode): - return cleaned - else: - return unicode(cleaned, 'UTF-8', 'replace') - return '' - - -class Base(object): - """ - This is just here to have an isinstance check, which is also used on - evaluate classes. But since they have sometimes a special type of - delegation, it is important for those classes to override this method. - - I know that there is a chance to do such things with __instancecheck__, but - since Python 2.5 doesn't support it, I decided to do it this way. - """ - __slots__ = () - - def isinstance(self, *cls): - return isinstance(self, cls) - - @Python3Method - def get_parent_until(self, classes=(), reverse=False, - include_current=True): - """ - Searches the parent "chain" until the object is an instance of - classes. If classes is empty return the last parent in the chain - (is without a parent). - """ - if type(classes) not in (tuple, list): - classes = (classes,) - scope = self if include_current else self.parent - while scope.parent is not None: - # TODO why if classes? - if classes and reverse != scope.isinstance(*classes): - break - scope = scope.parent - return scope - - def get_parent_scope(self, include_flows=False): - """ - Returns the underlying scope. - """ - scope = self.parent - while scope is not None: - if include_flows and isinstance(scope, Flow): - return scope - if scope.is_scope(): - break - scope = scope.parent - return scope - - def get_definition(self): - if self.type in ('newline', 'dedent', 'indent', 'endmarker'): - raise ValueError('Cannot get the indentation of whitespace or indentation.') - scope = self - while scope.parent is not None: - parent = scope.parent - if scope.isinstance(Node, Leaf) and parent.type != 'simple_stmt': - if scope.type == 'testlist_comp': - try: - if isinstance(scope.children[1], CompFor): - return scope.children[1] - except IndexError: - pass - scope = parent - else: - break - return scope - - def assignment_indexes(self): - """ - Returns an array of tuple(int, node) of the indexes that are used in - tuple assignments. - - For example if the name is ``y`` in the following code:: - - x, (y, z) = 2, '' - - would result in ``[(1, xyz_node), (0, yz_node)]``. - """ - indexes = [] - node = self.parent - compare = self - while node is not None: - if is_node(node, 'testlist_comp', 'testlist_star_expr', 'exprlist'): - for i, child in enumerate(node.children): - if child == compare: - indexes.insert(0, (int(i / 2), node)) - break - else: - raise LookupError("Couldn't find the assignment.") - elif isinstance(node, (ExprStmt, CompFor)): - break - - compare = node - node = node.parent - return indexes - - def is_scope(self): - # Default is not being a scope. Just inherit from Scope. - return False - - @abc.abstractmethod - def nodes_to_execute(self, last_added=False): - raise NotImplementedError() - - def get_next_sibling(self): - """ - The node immediately following the invocant in their parent's children - list. If the invocant does not have a next sibling, it is None - """ - # Can't use index(); we need to test by identity - for i, child in enumerate(self.parent.children): - if child is self: - try: - return self.parent.children[i + 1] - except IndexError: - return None - - def get_previous_sibling(self): - """ - The node/leaf immediately preceding the invocant in their parent's - children list. If the invocant does not have a previous sibling, it is - None. - """ - # Can't use index(); we need to test by identity - for i, child in enumerate(self.parent.children): - if child is self: - if i == 0: - return None - return self.parent.children[i - 1] - - def get_previous_leaf(self): - """ - Returns the previous leaf in the parser tree. - Raises an IndexError if it's the first element. - """ - node = self - while True: - c = node.parent.children - i = c.index(node) - if i == 0: - node = node.parent - if node.parent is None: - raise IndexError('Cannot access the previous element of the first one.') - else: - node = c[i - 1] - break - - while True: - try: - node = node.children[-1] - except AttributeError: # A Leaf doesn't have children. - return node - - def get_next_leaf(self): - """ - Returns the previous leaf in the parser tree. - Raises an IndexError if it's the last element. - """ - node = self - while True: - c = node.parent.children - i = c.index(node) - if i == len(c) - 1: - node = node.parent - if node.parent is None: - raise IndexError('Cannot access the next element of the last one.') - else: - node = c[i + 1] - break - - while True: - try: - node = node.children[0] - except AttributeError: # A Leaf doesn't have children. - return node - - -class Leaf(Base): - __slots__ = ('position_modifier', 'value', 'parent', '_start_pos', 'prefix') - - def __init__(self, position_modifier, value, start_pos, prefix=''): - self.position_modifier = position_modifier - self.value = value - self._start_pos = start_pos - self.prefix = prefix - self.parent = None - - @property - def start_pos(self): - return self._start_pos[0] + self.position_modifier.line, self._start_pos[1] - - @start_pos.setter - def start_pos(self, value): - self._start_pos = value[0] - self.position_modifier.line, value[1] - - def get_start_pos_of_prefix(self): - try: - previous_leaf = self - while True: - previous_leaf = previous_leaf.get_previous_leaf() - if previous_leaf.type not in ('indent', 'dedent'): - return previous_leaf.end_pos - except IndexError: - return 1, 0 # It's the first leaf. - - @property - def end_pos(self): - return (self._start_pos[0] + self.position_modifier.line, - self._start_pos[1] + len(self.value)) - - def move(self, line_offset, column_offset): - self._start_pos = (self._start_pos[0] + line_offset, - self._start_pos[1] + column_offset) - - def first_leaf(self): - return self - - def get_code(self, normalized=False, include_prefix=True): - if normalized: - return self.value - if include_prefix: - return self.prefix + self.value - else: - return self.value - - def nodes_to_execute(self, last_added=False): - return [] - - @utf8_repr - def __repr__(self): - return "<%s: %s>" % (type(self).__name__, self.value) - - -class LeafWithNewLines(Leaf): - __slots__ = () - - @property - def end_pos(self): - """ - Literals and whitespace end_pos are more complicated than normal - end_pos, because the containing newlines may change the indexes. - """ - end_pos_line, end_pos_col = self.start_pos - lines = self.value.split('\n') - end_pos_line += len(lines) - 1 - # Check for multiline token - if self.start_pos[0] == end_pos_line: - end_pos_col += len(lines[-1]) - else: - end_pos_col = len(lines[-1]) - return end_pos_line, end_pos_col - - @utf8_repr - def __repr__(self): - return "<%s: %r>" % (type(self).__name__, self.value) - - -class EndMarker(Leaf): - __slots__ = () - type = 'endmarker' - - -class Newline(LeafWithNewLines): - """Contains NEWLINE and ENDMARKER tokens.""" - __slots__ = () - type = 'newline' - - @utf8_repr - def __repr__(self): - return "<%s: %s>" % (type(self).__name__, repr(self.value)) - - -class Name(Leaf): - """ - A string. Sometimes it is important to know if the string belongs to a name - or not. - """ - type = 'name' - __slots__ = () - - def __str__(self): - return self.value - - def __unicode__(self): - return self.value - - def __repr__(self): - return "<%s: %s@%s,%s>" % (type(self).__name__, self.value, - self.start_pos[0], self.start_pos[1]) - - def is_definition(self): - if self.parent.type in ('power', 'atom_expr'): - # In `self.x = 3` self is not a definition, but x is. - return False - - stmt = self.get_definition() - if stmt.type in ('funcdef', 'classdef', 'file_input', 'param'): - return self == stmt.name - elif stmt.type == 'for_stmt': - return self.start_pos < stmt.children[2].start_pos - elif stmt.type == 'try_stmt': - return self.get_previous_sibling() == 'as' - else: - return stmt.type in ('expr_stmt', 'import_name', 'import_from', - 'comp_for', 'with_stmt') \ - and self in stmt.get_defined_names() - - def nodes_to_execute(self, last_added=False): - if last_added is False: - yield self - - -class Literal(LeafWithNewLines): - __slots__ = () - - def eval(self): - return literal_eval(self.value) - - -class Number(Literal): - type = 'number' - __slots__ = () - - -class String(Literal): - type = 'string' - __slots__ = () - - -class Indent(Leaf): - type = 'indent' - __slots__ = () - - -class Dedent(Leaf): - type = 'dedent' - __slots__ = () - - -class Operator(Leaf): - type = 'operator' - __slots__ = () - - def __str__(self): - return self.value - - def __eq__(self, other): - """ - Make comparisons with strings easy. - Improves the readability of the parser. - """ - if isinstance(other, Operator): - return self is other - else: - return self.value == other - - def __ne__(self, other): - """Python 2 compatibility.""" - return self.value != other - - def __hash__(self): - return hash(self.value) - - -class Keyword(Leaf): - type = 'keyword' - __slots__ = () - - def __eq__(self, other): - """ - Make comparisons with strings easy. - Improves the readability of the parser. - """ - if isinstance(other, Keyword): - return self is other - return self.value == other - - def __ne__(self, other): - """Python 2 compatibility.""" - return not self.__eq__(other) - - def __hash__(self): - return hash(self.value) - - -class BaseNode(Base): - """ - The super class for Scope, Import, Name and Statement. Every object in - the parser tree inherits from this class. - """ - __slots__ = ('children', 'parent') - type = None - - def __init__(self, children): - """ - Initialize :class:`BaseNode`. - - :param children: The module in which this Python object locates. - """ - for c in children: - c.parent = self - self.children = children - self.parent = None - - def move(self, line_offset, column_offset): - """ - Move the Node's start_pos. - """ - for c in self.children: - c.move(line_offset, column_offset) - - @property - def start_pos(self): - return self.children[0].start_pos - - def get_start_pos_of_prefix(self): - return self.children[0].get_start_pos_of_prefix() - - @property - def end_pos(self): - return self.children[-1].end_pos - - def get_code(self, normalized=False, include_prefix=True): - # TODO implement normalized (depending on context). - if include_prefix: - return "".join(c.get_code(normalized) for c in self.children) - else: - first = self.children[0].get_code(include_prefix=False) - return first + "".join(c.get_code(normalized) for c in self.children[1:]) - - @Python3Method - def name_for_position(self, position): - for c in self.children: - if isinstance(c, Leaf): - if isinstance(c, Name) and c.start_pos <= position <= c.end_pos: - return c - else: - result = c.name_for_position(position) - if result is not None: - return result - return None - - def get_leaf_for_position(self, position, include_prefixes=False): - for c in self.children: - if include_prefixes: - start_pos = c.get_start_pos_of_prefix() - else: - start_pos = c.start_pos - - if start_pos <= position <= c.end_pos: - try: - return c.get_leaf_for_position(position, include_prefixes) - except AttributeError: - while c.type in ('indent', 'dedent'): - # We'd rather not have indents and dedents as a leaf, - # because they don't contain indentation information. - c = c.get_next_leaf() - return c - - return None - - @Python3Method - def get_statement_for_position(self, pos): - for c in self.children: - if c.start_pos <= pos <= c.end_pos: - if c.type not in ('decorated', 'simple_stmt', 'suite') \ - and not isinstance(c, (Flow, ClassOrFunc)): - return c - else: - try: - return c.get_statement_for_position(pos) - except AttributeError: - pass # Must be a non-scope - return None - - def first_leaf(self): - try: - return self.children[0].first_leaf() - except AttributeError: - return self.children[0] - - def get_next_leaf(self): - """ - Raises an IndexError if it's the last node. (Would be the module) - """ - c = self.parent.children - index = c.index(self) - if index == len(c) - 1: - # TODO WTF? recursion? - return self.get_next_leaf() - else: - return c[index + 1] - - def last_leaf(self): - try: - return self.children[-1].last_leaf() - except AttributeError: - return self.children[-1] - - def get_following_comment_same_line(self): - """ - returns (as string) any comment that appears on the same line, - after the node, including the # - """ - try: - if self.isinstance(ForStmt): - whitespace = self.children[5].first_leaf().prefix - elif self.isinstance(WithStmt): - whitespace = self.children[3].first_leaf().prefix - else: - whitespace = self.last_leaf().get_next_leaf().prefix - except AttributeError: - return None - except ValueError: - # TODO in some particular cases, the tree doesn't seem to be linked - # correctly - return None - if "#" not in whitespace: - return None - comment = whitespace[whitespace.index("#"):] - if "\r" in comment: - comment = comment[:comment.index("\r")] - if "\n" in comment: - comment = comment[:comment.index("\n")] - return comment - - @utf8_repr - def __repr__(self): - code = self.get_code().replace('\n', ' ').strip() - if not is_py3: - code = code.encode(encoding, 'replace') - return "<%s: %s@%s,%s>" % \ - (type(self).__name__, code, self.start_pos[0], self.start_pos[1]) - - -class Node(BaseNode): - """Concrete implementation for interior nodes.""" - __slots__ = ('type',) - - _IGNORE_EXECUTE_NODES = set([ - 'suite', 'subscriptlist', 'subscript', 'simple_stmt', 'sliceop', - 'testlist_comp', 'dictorsetmaker', 'trailer', 'decorators', - 'decorated', 'arglist', 'argument', 'exprlist', 'testlist', - 'testlist_safe', 'testlist1' - ]) - - def __init__(self, type, children): - """ - Initializer. - - Takes a type constant (a symbol number >= 256), a sequence of - child nodes, and an optional context keyword argument. - - As a side effect, the parent pointers of the children are updated. - """ - super(Node, self).__init__(children) - self.type = type - - def nodes_to_execute(self, last_added=False): - """ - For static analysis. - """ - result = [] - if self.type not in Node._IGNORE_EXECUTE_NODES and not last_added: - result.append(self) - last_added = True - - for child in self.children: - result += child.nodes_to_execute(last_added) - return result - - def __repr__(self): - return "%s(%s, %r)" % (self.__class__.__name__, self.type, self.children) - - -class ErrorNode(BaseNode): - """ - TODO doc - """ - __slots__ = () - type = 'error_node' - - def nodes_to_execute(self, last_added=False): - return [] - - -class ErrorLeaf(LeafWithNewLines): - """ - TODO doc - """ - __slots__ = ('original_type') - type = 'error_leaf' - - def __init__(self, position_modifier, original_type, value, start_pos, prefix=''): - super(ErrorLeaf, self).__init__(position_modifier, value, start_pos, prefix) - self.original_type = original_type - - def __repr__(self): - token_type = token.tok_name[self.original_type] - return "<%s: %s, %s)>" % (type(self).__name__, token_type, self.start_pos) - - -class IsScopeMeta(type): - def __instancecheck__(self, other): - return other.is_scope() - - -class IsScope(use_metaclass(IsScopeMeta)): - pass - - -class Scope(BaseNode, DocstringMixin): - """ - Super class for the parser tree, which represents the state of a python - text file. - A Scope manages and owns its subscopes, which are classes and functions, as - well as variables and imports. It is used to access the structure of python - files. - - :param start_pos: The position (line and column) of the scope. - :type start_pos: tuple(int, int) - """ - __slots__ = ('names_dict',) - - def __init__(self, children): - super(Scope, self).__init__(children) - - @property - def returns(self): - # Needed here for fast_parser, because the fast_parser splits and - # returns will be in "normal" modules. - return self._search_in_scope(ReturnStmt) - - @property - def subscopes(self): - return self._search_in_scope(Scope) - - @property - def flows(self): - return self._search_in_scope(Flow) - - @property - def imports(self): - return self._search_in_scope(Import) - - @Python3Method - def _search_in_scope(self, typ): - def scan(children): - elements = [] - for element in children: - if isinstance(element, typ): - elements.append(element) - if is_node(element, 'suite', 'simple_stmt', 'decorated') \ - or isinstance(element, Flow): - elements += scan(element.children) - return elements - - return scan(self.children) - - @property - def statements(self): - return self._search_in_scope((ExprStmt, KeywordStatement)) - - def is_scope(self): - return True - - def __repr__(self): - try: - name = self.path - except AttributeError: - try: - name = self.name - except AttributeError: - name = self.command - - return "<%s: %s@%s-%s>" % (type(self).__name__, name, - self.start_pos[0], self.end_pos[0]) - - def walk(self): - yield self - for s in self.subscopes: - for scope in s.walk(): - yield scope - - for r in self.statements: - while isinstance(r, Flow): - for scope in r.walk(): - yield scope - r = r.next - - -class Module(Scope): - """ - The top scope, which is always a module. - Depending on the underlying parser this may be a full module or just a part - of a module. - """ - __slots__ = ('path', 'global_names', 'used_names', '_name') - type = 'file_input' - - def __init__(self, children): - """ - Initialize :class:`Module`. - - :type path: str - :arg path: File path to this module. - - .. todo:: Document `top_module`. - """ - super(Module, self).__init__(children) - self.path = None # Set later. - - @property - @underscore_memoization - def name(self): - """ This is used for the goto functions. """ - if self.path is None: - string = '' # no path -> empty name - else: - sep = (re.escape(os.path.sep),) * 2 - r = re.search(r'([^%s]*?)(%s__init__)?(\.py|\.so)?$' % sep, self.path) - # Remove PEP 3149 names - string = re.sub('\.[a-z]+-\d{2}[mud]{0,3}$', '', r.group(1)) - # Positions are not real, but a module starts at (1, 0) - p = (1, 0) - name = Name(zero_position_modifier, string, p) - name.parent = self - return name - - @property - def has_explicit_absolute_import(self): - """ - Checks if imports in this module are explicitly absolute, i.e. there - is a ``__future__`` import. - """ - # TODO this is a strange scan and not fully correct. I think Python's - # parser does it in a different way and scans for the first - # statement/import with a tokenizer (to check for syntax changes like - # the future print statement). - for imp in self.imports: - if imp.type == 'import_from' and imp.level == 0: - for path in imp.paths(): - if [str(name) for name in path] == ['__future__', 'absolute_import']: - return True - return False - - def nodes_to_execute(self, last_added=False): - # Yield itself, class needs to be executed for decorator checks. - result = [] - for child in self.children: - result += child.nodes_to_execute() - return result - - -class Decorator(BaseNode): - type = 'decorator' - __slots__ = () - - def nodes_to_execute(self, last_added=False): - if self.children[-2] == ')': - node = self.children[-3] - if node != '(': - return node.nodes_to_execute() - return [] - - -class ClassOrFunc(Scope): - __slots__ = () - - @property - def name(self): - return self.children[1] - - def get_decorators(self): - decorated = self.parent - if is_node(decorated, 'decorated'): - if is_node(decorated.children[0], 'decorators'): - return decorated.children[0].children - else: - return decorated.children[:1] - else: - return [] - - -class Class(ClassOrFunc): - """ - Used to store the parsed contents of a python class. - - :param name: The Class name. - :type name: str - :param supers: The super classes of a Class. - :type supers: list - :param start_pos: The start position (line, column) of the class. - :type start_pos: tuple(int, int) - """ - type = 'classdef' - __slots__ = () - - def __init__(self, children): - super(Class, self).__init__(children) - - def get_super_arglist(self): - if self.children[2] != '(': # Has no parentheses - return None - else: - if self.children[3] == ')': # Empty parentheses - return None - else: - return self.children[3] - - @property - def doc(self): - """ - Return a document string including call signature of __init__. - """ - docstr = self.raw_doc - for sub in self.subscopes: - if str(sub.name) == '__init__': - return '%s\n\n%s' % ( - sub.get_call_signature(func_name=self.name), docstr) - return docstr - - def nodes_to_execute(self, last_added=False): - # Yield itself, class needs to be executed for decorator checks. - yield self - # Super arguments. - arglist = self.get_super_arglist() - try: - children = arglist.children - except AttributeError: - if arglist is not None: - for node_to_execute in arglist.nodes_to_execute(): - yield node_to_execute - else: - for argument in children: - if argument.type == 'argument': - # metaclass= or list comprehension or */** - raise NotImplementedError('Metaclasses not implemented') - else: - for node_to_execute in argument.nodes_to_execute(): - yield node_to_execute - - # care for the class suite: - for node in self.children[self.children.index(':'):]: - # This could be easier without the fast parser. But we need to find - # the position of the colon, because everything after it can be a - # part of the class, not just its suite. - for node_to_execute in node.nodes_to_execute(): - yield node_to_execute - - -def _create_params(parent, argslist_list): - """ - `argslist_list` is a list that can contain an argslist as a first item, but - most not. It's basically the items between the parameter brackets (which is - at most one item). - This function modifies the parser structure. It generates `Param` objects - from the normal ast. Those param objects do not exist in a normal ast, but - make the evaluation of the ast tree so much easier. - You could also say that this function replaces the argslist node with a - list of Param objects. - """ - def check_python2_nested_param(node): - """ - Python 2 allows params to look like ``def x(a, (b, c))``, which is - basically a way of unpacking tuples in params. Python 3 has ditched - this behavior. Jedi currently just ignores those constructs. - """ - return node.type == 'tfpdef' and node.children[0] == '(' - - try: - first = argslist_list[0] - except IndexError: - return [] - - if first.type in ('name', 'tfpdef'): - if check_python2_nested_param(first): - return [first] - else: - return [Param([first], parent)] - elif first == '*': - return [first] - else: # argslist is a `typedargslist` or a `varargslist`. - children = first.children - new_children = [] - start = 0 - # Start with offset 1, because the end is higher. - for end, child in enumerate(children + [None], 1): - if child is None or child == ',': - param_children = children[start:end] - if param_children: # Could as well be comma and then end. - if check_python2_nested_param(param_children[0]): - new_children += param_children - elif param_children[0] == '*' and param_children[1] == ',': - new_children += param_children - else: - new_children.append(Param(param_children, parent)) - start = end - return new_children - - -class Function(ClassOrFunc): - """ - Used to store the parsed contents of a python function. - - Children: - 0) - 1) - 2) parameter list (including open-paren and close-paren s) - 3) - 4) Node() representing function body - 5) ?? - 6) annotation (if present) - """ - __slots__ = ('listeners',) - type = 'funcdef' - - def __init__(self, children): - super(Function, self).__init__(children) - self.listeners = set() # not used here, but in evaluation. - parameters = self.children[2] # After `def foo` - parameters.children[1:-1] = _create_params(parameters, parameters.children[1:-1]) - - @property - def params(self): - return [p for p in self.children[2].children if p.type == 'param'] - - @property - def name(self): - return self.children[1] # First token after `def` - - @property - def yields(self): - # TODO This is incorrect, yields are also possible in a statement. - return self._search_in_scope(YieldExpr) - - def is_generator(self): - return bool(self.yields) - - def annotation(self): - try: - if self.children[3] == "->": - return self.children[4] - assert self.children[3] == ":" - return None - except IndexError: - return None - - def get_call_signature(self, width=72, func_name=None): - """ - Generate call signature of this function. - - :param width: Fold lines if a line is longer than this value. - :type width: int - :arg func_name: Override function name when given. - :type func_name: str - - :rtype: str - """ - func_name = func_name or self.name - code = unicode(func_name) + self._get_paramlist_code() - return '\n'.join(textwrap.wrap(code, width)) - - def _get_paramlist_code(self): - return self.children[2].get_code() - - @property - def doc(self): - """ Return a document string including call signature. """ - docstr = self.raw_doc - return '%s\n\n%s' % (self.get_call_signature(), docstr) - - def nodes_to_execute(self, last_added=False): - # Yield itself, functions needs to be executed for decorator checks. - yield self - for param in self.params: - if param.default is not None: - yield param.default - # care for the function suite: - for node in self.children[4:]: - # This could be easier without the fast parser. The fast parser - # allows that the 4th position is empty or that there's even a - # fifth element (another function/class). So just scan everything - # after colon. - for node_to_execute in node.nodes_to_execute(): - yield node_to_execute - - -class Lambda(Function): - """ - Lambdas are basically trimmed functions, so give it the same interface. - - Children: - 0) - *) for each argument x - -2) - -1) Node() representing body - """ - type = 'lambda' - __slots__ = () - - def __init__(self, children): - # We don't want to call the Function constructor, call its parent. - super(Function, self).__init__(children) - self.listeners = set() # not used here, but in evaluation. - lst = self.children[1:-2] # Everything between `lambda` and the `:` operator is a parameter. - self.children[1:-2] = _create_params(self, lst) - - @property - def name(self): - # Borrow the position of the AST node. - return Name(self.children[0].position_modifier, '', self.children[0].start_pos) - - def _get_paramlist_code(self): - return '(' + ''.join(param.get_code() for param in self.params).strip() + ')' - - @property - def params(self): - return self.children[1:-2] - - def is_generator(self): - return False - - def annotation(self): - # lambda functions do not support annotations - return None - - @property - def yields(self): - return [] - - def nodes_to_execute(self, last_added=False): - for param in self.params: - if param.default is not None: - yield param.default - # Care for the lambda test (last child): - for node_to_execute in self.children[-1].nodes_to_execute(): - yield node_to_execute - - def __repr__(self): - return "<%s@%s>" % (self.__class__.__name__, self.start_pos) - - -class Flow(BaseNode): - __slots__ = () - - def nodes_to_execute(self, last_added=False): - for child in self.children: - for node_to_execute in child.nodes_to_execute(): - yield node_to_execute - - -class IfStmt(Flow): - type = 'if_stmt' - __slots__ = () - - def check_nodes(self): - """ - Returns all the `test` nodes that are defined as x, here: - - if x: - pass - elif x: - pass - """ - for i, c in enumerate(self.children): - if c in ('elif', 'if'): - yield self.children[i + 1] - - def node_in_which_check_node(self, node): - """ - Returns the check node (see function above) that a node is contained - in. However if it the node is in the check node itself and not in the - suite return None. - """ - start_pos = node.start_pos - for check_node in reversed(list(self.check_nodes())): - if check_node.start_pos < start_pos: - if start_pos < check_node.end_pos: - return None - # In this case the node is within the check_node itself, - # not in the suite - else: - return check_node - - def node_after_else(self, node): - """ - Checks if a node is defined after `else`. - """ - for c in self.children: - if c == 'else': - if node.start_pos > c.start_pos: - return True - else: - return False - - -class WhileStmt(Flow): - type = 'while_stmt' - __slots__ = () - - -class ForStmt(Flow): - type = 'for_stmt' - __slots__ = () - - def get_input_node(self): - """ - Returns the input node ``y`` from: ``for x in y:``. - """ - return self.children[3] - - def defines_one_name(self): - """ - Returns True if only one name is returned: ``for x in y``. - Returns False if the for loop is more complicated: ``for x, z in y``. - - :returns: bool - """ - return self.children[1].type == 'name' - - -class TryStmt(Flow): - type = 'try_stmt' - __slots__ = () - - def except_clauses(self): - """ - Returns the ``test`` nodes found in ``except_clause`` nodes. - Returns ``[None]`` for except clauses without an exception given. - """ - for node in self.children: - if node.type == 'except_clause': - yield node.children[1] - elif node == 'except': - yield None - - def nodes_to_execute(self, last_added=False): - result = [] - for child in self.children[2::3]: - result += child.nodes_to_execute() - for child in self.children[0::3]: - if child.type == 'except_clause': - # Add the test node and ignore the `as NAME` definition. - result += child.children[1].nodes_to_execute() - return result - - -class WithStmt(Flow): - type = 'with_stmt' - __slots__ = () - - def get_defined_names(self): - names = [] - for with_item in self.children[1:-2:2]: - # Check with items for 'as' names. - if is_node(with_item, 'with_item'): - names += _defined_names(with_item.children[2]) - return names - - def node_from_name(self, name): - node = name - while True: - node = node.parent - if is_node(node, 'with_item'): - return node.children[0] - - def nodes_to_execute(self, last_added=False): - result = [] - for child in self.children[1::2]: - if child.type == 'with_item': - # Just ignore the `as EXPR` part - at least for now, because - # most times it's just a name. - child = child.children[0] - result += child.nodes_to_execute() - return result - - -class Import(BaseNode): - __slots__ = () - - def path_for_name(self, name): - try: - # The name may be an alias. If it is, just map it back to the name. - name = self.aliases()[name] - except KeyError: - pass - - for path in self.paths(): - if name in path: - return path[:path.index(name) + 1] - raise ValueError('Name should be defined in the import itself') - - def is_nested(self): - return False # By default, sub classes may overwrite this behavior - - def is_star_import(self): - return self.children[-1] == '*' - - def nodes_to_execute(self, last_added=False): - """ - `nodes_to_execute` works a bit different for imports, because the names - itself cannot directly get resolved (except on itself). - """ - # TODO couldn't we return the names? Would be nicer. - return [self] - - -class ImportFrom(Import): - type = 'import_from' - __slots__ = () - - def get_defined_names(self): - return [alias or name for name, alias in self._as_name_tuples()] - - def aliases(self): - """Mapping from alias to its corresponding name.""" - return dict((alias, name) for name, alias in self._as_name_tuples() - if alias is not None) - - def get_from_names(self): - for n in self.children[1:]: - if n not in ('.', '...'): - break - if is_node(n, 'dotted_name'): # from x.y import - return n.children[::2] - elif n == 'import': # from . import - return [] - else: # from x import - return [n] - - @property - def level(self): - """The level parameter of ``__import__``.""" - level = 0 - for n in self.children[1:]: - if n in ('.', '...'): - level += len(n.value) - else: - break - return level - - def _as_name_tuples(self): - last = self.children[-1] - if last == ')': - last = self.children[-2] - elif last == '*': - return # No names defined directly. - - if is_node(last, 'import_as_names'): - as_names = last.children[::2] - else: - as_names = [last] - for as_name in as_names: - if as_name.type == 'name': - yield as_name, None - else: - yield as_name.children[::2] # yields x, y -> ``x as y`` - - def star_import_name(self): - """ - The last name defined in a star import. - """ - return self.paths()[-1][-1] - - def paths(self): - """ - The import paths defined in an import statement. Typically an array - like this: ``[, ]``. - """ - dotted = self.get_from_names() - - if self.children[-1] == '*': - return [dotted] - return [dotted + [name] for name, alias in self._as_name_tuples()] - - -class ImportName(Import): - """For ``import_name`` nodes. Covers normal imports without ``from``.""" - type = 'import_name' - __slots__ = () - - def get_defined_names(self): - return [alias or path[0] for path, alias in self._dotted_as_names()] - - @property - def level(self): - """The level parameter of ``__import__``.""" - return 0 # Obviously 0 for imports without from. - - def paths(self): - return [path for path, alias in self._dotted_as_names()] - - def _dotted_as_names(self): - """Generator of (list(path), alias) where alias may be None.""" - dotted_as_names = self.children[1] - if is_node(dotted_as_names, 'dotted_as_names'): - as_names = dotted_as_names.children[::2] - else: - as_names = [dotted_as_names] - - for as_name in as_names: - if is_node(as_name, 'dotted_as_name'): - alias = as_name.children[2] - as_name = as_name.children[0] - else: - alias = None - if as_name.type == 'name': - yield [as_name], alias - else: - # dotted_names - yield as_name.children[::2], alias - - def is_nested(self): - """ - This checks for the special case of nested imports, without aliases and - from statement:: - - import foo.bar - """ - return [1 for path, alias in self._dotted_as_names() - if alias is None and len(path) > 1] - - def aliases(self): - return dict((alias, path[-1]) for path, alias in self._dotted_as_names() - if alias is not None) - - -class KeywordStatement(BaseNode): - """ - For the following statements: `assert`, `del`, `global`, `nonlocal`, - `raise`, `return`, `yield`, `return`, `yield`. - - `pass`, `continue` and `break` are not in there, because they are just - simple keywords and the parser reduces it to a keyword. - """ - __slots__ = () - - @property - def type(self): - """ - Keyword statements start with the keyword and end with `_stmt`. You can - crosscheck this with the Python grammar. - """ - return '%s_stmt' % self.keyword - - @property - def keyword(self): - return self.children[0].value - - def nodes_to_execute(self, last_added=False): - result = [] - for child in self.children: - result += child.nodes_to_execute() - return result - - -class AssertStmt(KeywordStatement): - __slots__ = () - - def assertion(self): - return self.children[1] - - -class GlobalStmt(KeywordStatement): - __slots__ = () - - def get_defined_names(self): - return [] - - def get_global_names(self): - return self.children[1::2] - - def nodes_to_execute(self, last_added=False): - """ - The global keyword allows to define any name. Even if it doesn't - exist. - """ - return [] - - -class ReturnStmt(KeywordStatement): - __slots__ = () - - -class YieldExpr(BaseNode): - __slots__ = () - - @property - def type(self): - return 'yield_expr' - - def nodes_to_execute(self, last_added=False): - if len(self.children) > 1: - return self.children[1].nodes_to_execute() - else: - return [] - - -def _defined_names(current): - """ - A helper function to find the defined names in statements, for loops and - list comprehensions. - """ - names = [] - if is_node(current, 'testlist_star_expr', 'testlist_comp', 'exprlist'): - for child in current.children[::2]: - names += _defined_names(child) - elif is_node(current, 'atom', 'star_expr'): - names += _defined_names(current.children[1]) - elif is_node(current, 'power', 'atom_expr'): - if current.children[-2] != '**': # Just if there's no operation - trailer = current.children[-1] - if trailer.children[0] == '.': - names.append(trailer.children[1]) - else: - names.append(current) - return names - - -class ExprStmt(BaseNode, DocstringMixin): - type = 'expr_stmt' - __slots__ = () - - def get_defined_names(self): - names = [] - if self.children[1].type == 'annassign': - names = _defined_names(self.children[0]) - return list(chain.from_iterable( - _defined_names(self.children[i]) - for i in range(0, len(self.children) - 2, 2) - if '=' in self.children[i + 1].value) - ) + names - - def get_rhs(self): - """Returns the right-hand-side of the equals.""" - return self.children[-1] - - def first_operation(self): - """ - Returns `+=`, `=`, etc or None if there is no operation. - """ - try: - return self.children[1] - except IndexError: - return None - - def nodes_to_execute(self, last_added=False): - # I think evaluating the statement (and possibly returned arrays), - # should be enough for static analysis. - result = [self] - for child in self.children: - result += child.nodes_to_execute(last_added=True) - return result - - -class Param(BaseNode): - """ - It's a helper class that makes business logic with params much easier. The - Python grammar defines no ``param`` node. It defines it in a different way - that is not really suited to working with parameters. - """ - type = 'param' - - def __init__(self, children, parent): - super(Param, self).__init__(children) - self.parent = parent - for child in children: - child.parent = self - - @property - def stars(self): - first = self.children[0] - if first in ('*', '**'): - return len(first.value) - return 0 - - @property - def default(self): - try: - return self.children[int(self.children[0] in ('*', '**')) + 2] - except IndexError: - return None - - def annotation(self): - tfpdef = self._tfpdef() - if is_node(tfpdef, 'tfpdef'): - assert tfpdef.children[1] == ":" - assert len(tfpdef.children) == 3 - annotation = tfpdef.children[2] - return annotation - else: - return None - - def _tfpdef(self): - """ - tfpdef: see grammar.txt. - """ - offset = int(self.children[0] in ('*', '**')) - return self.children[offset] - - @property - def name(self): - if is_node(self._tfpdef(), 'tfpdef'): - return self._tfpdef().children[0] - else: - return self._tfpdef() - - @property - def position_nr(self): - return self.parent.children.index(self) - 1 - - @property - def parent_function(self): - return self.get_parent_until(IsScope) - - def __repr__(self): - default = '' if self.default is None else '=%s' % self.default - return '<%s: %s>' % (type(self).__name__, str(self._tfpdef()) + default) - - -class CompFor(BaseNode): - type = 'comp_for' - __slots__ = () - - def get_comp_fors(self): - yield self - last = self.children[-1] - while True: - if isinstance(last, CompFor): - yield last - elif not is_node(last, 'comp_if'): - break - last = last.children[-1] - - def is_scope(self): - return True - - @property - def names_dict(self): - dct = {} - for name in self.get_defined_names(): - arr = dct.setdefault(name.value, []) - arr.append(name) - return dct - - def names_dicts(self, search_global): - yield self.names_dict - - def get_defined_names(self): - return _defined_names(self.children[1]) - - def nodes_to_execute(self, last_added=False): - last = self.children[-1] - if last.type == 'comp_if': - for node in last.children[-1].nodes_to_execute(): - yield node - last = self.children[-2] - elif last.type == 'comp_for': - for node in last.nodes_to_execute(): - yield node - last = self.children[-2] - for node in last.nodes_to_execute(): - yield node diff --git a/pythonFiles/preview/jedi/parser/utils.py b/pythonFiles/preview/jedi/parser/utils.py deleted file mode 100644 index 9098b095e6d6..000000000000 --- a/pythonFiles/preview/jedi/parser/utils.py +++ /dev/null @@ -1,198 +0,0 @@ -import inspect -import time -import os -import sys -import json -import hashlib -import gc -import shutil -import pickle - -from jedi import settings -from jedi import debug - - -def underscore_memoization(func): - """ - Decorator for methods:: - - class A(object): - def x(self): - if self._x: - self._x = 10 - return self._x - - Becomes:: - - class A(object): - @underscore_memoization - def x(self): - return 10 - - A now has an attribute ``_x`` written by this decorator. - """ - name = '_' + func.__name__ - - def wrapper(self): - try: - return getattr(self, name) - except AttributeError: - result = func(self) - if inspect.isgenerator(result): - result = list(result) - setattr(self, name, result) - return result - - return wrapper - - -# for fast_parser, should not be deleted -parser_cache = {} - - -class ParserCacheItem(object): - def __init__(self, parser, change_time=None): - self.parser = parser - if change_time is None: - change_time = time.time() - self.change_time = change_time - - -def load_parser(path): - """ - Returns the module or None, if it fails. - """ - p_time = os.path.getmtime(path) if path else None - try: - parser_cache_item = parser_cache[path] - if not path or p_time <= parser_cache_item.change_time: - return parser_cache_item.parser - except KeyError: - if settings.use_filesystem_cache: - return ParserPickling.load_parser(path, p_time) - - -def save_parser(path, parser, pickling=True): - try: - p_time = None if path is None else os.path.getmtime(path) - except OSError: - p_time = None - pickling = False - - item = ParserCacheItem(parser, p_time) - parser_cache[path] = item - if settings.use_filesystem_cache and pickling: - ParserPickling.save_parser(path, item) - - -class ParserPickling(object): - - version = 26 - """ - Version number (integer) for file system cache. - - Increment this number when there are any incompatible changes in - parser representation classes. For example, the following changes - are regarded as incompatible. - - - Class name is changed. - - Class is moved to another module. - - Defined slot of the class is changed. - """ - - def __init__(self): - self.__index = None - self.py_tag = 'cpython-%s%s' % sys.version_info[:2] - """ - Short name for distinguish Python implementations and versions. - - It's like `sys.implementation.cache_tag` but for Python < 3.3 - we generate something similar. See: - http://docs.python.org/3/library/sys.html#sys.implementation - - .. todo:: Detect interpreter (e.g., PyPy). - """ - - def load_parser(self, path, original_changed_time): - try: - pickle_changed_time = self._index[path] - except KeyError: - return None - if original_changed_time is not None \ - and pickle_changed_time < original_changed_time: - # the pickle file is outdated - return None - - with open(self._get_hashed_path(path), 'rb') as f: - try: - gc.disable() - parser_cache_item = pickle.load(f) - finally: - gc.enable() - - debug.dbg('pickle loaded: %s', path) - parser_cache[path] = parser_cache_item - return parser_cache_item.parser - - def save_parser(self, path, parser_cache_item): - self.__index = None - try: - files = self._index - except KeyError: - files = {} - self._index = files - - with open(self._get_hashed_path(path), 'wb') as f: - pickle.dump(parser_cache_item, f, pickle.HIGHEST_PROTOCOL) - files[path] = parser_cache_item.change_time - - self._flush_index() - - @property - def _index(self): - if self.__index is None: - try: - with open(self._get_path('index.json')) as f: - data = json.load(f) - except (IOError, ValueError): - self.__index = {} - else: - # 0 means version is not defined (= always delete cache): - if data.get('version', 0) != self.version: - self.clear_cache() - else: - self.__index = data['index'] - return self.__index - - def _remove_old_modules(self): - # TODO use - change = False - if change: - self._flush_index(self) - self._index # reload index - - def _flush_index(self): - data = {'version': self.version, 'index': self._index} - with open(self._get_path('index.json'), 'w') as f: - json.dump(data, f) - self.__index = None - - def clear_cache(self): - shutil.rmtree(self._cache_directory()) - self.__index = {} - - def _get_hashed_path(self, path): - return self._get_path('%s.pkl' % hashlib.md5(path.encode("utf-8")).hexdigest()) - - def _get_path(self, file): - dir = self._cache_directory() - if not os.path.exists(dir): - os.makedirs(dir) - return os.path.join(dir, file) - - def _cache_directory(self): - return os.path.join(settings.cache_directory, self.py_tag) - - -# is a singleton -ParserPickling = ParserPickling() diff --git a/pythonFiles/preview/jedi/refactoring.py b/pythonFiles/preview/jedi/refactoring.py deleted file mode 100644 index a342f08e2b8f..000000000000 --- a/pythonFiles/preview/jedi/refactoring.py +++ /dev/null @@ -1,202 +0,0 @@ -""" -Introduce some basic refactoring functions to |jedi|. This module is still in a -very early development stage and needs much testing and improvement. - -.. warning:: I won't do too much here, but if anyone wants to step in, please - do. Refactoring is none of my priorities - -It uses the |jedi| `API `_ and supports currently the -following functions (sometimes bug-prone): - -- rename -- extract variable -- inline variable -""" -import difflib - -from jedi import common -from jedi.evaluate import helpers -from jedi.parser import tree as pt - - -class Refactoring(object): - def __init__(self, change_dct): - """ - :param change_dct: dict(old_path=(new_path, old_lines, new_lines)) - """ - self.change_dct = change_dct - - def old_files(self): - dct = {} - for old_path, (new_path, old_l, new_l) in self.change_dct.items(): - dct[new_path] = '\n'.join(new_l) - return dct - - def new_files(self): - dct = {} - for old_path, (new_path, old_l, new_l) in self.change_dct.items(): - dct[new_path] = '\n'.join(new_l) - return dct - - def diff(self): - texts = [] - for old_path, (new_path, old_l, new_l) in self.change_dct.items(): - if old_path: - udiff = difflib.unified_diff(old_l, new_l) - else: - udiff = difflib.unified_diff(old_l, new_l, old_path, new_path) - texts.append('\n'.join(udiff)) - return '\n'.join(texts) - - -def rename(script, new_name): - """ The `args` / `kwargs` params are the same as in `api.Script`. - :param operation: The refactoring operation to execute. - :type operation: str - :type source: str - :return: list of changed lines/changed files - """ - return Refactoring(_rename(script.usages(), new_name)) - - -def _rename(names, replace_str): - """ For both rename and inline. """ - order = sorted(names, key=lambda x: (x.module_path, x.line, x.column), - reverse=True) - - def process(path, old_lines, new_lines): - if new_lines is not None: # goto next file, save last - dct[path] = path, old_lines, new_lines - - dct = {} - current_path = object() - new_lines = old_lines = None - for name in order: - if name.in_builtin_module(): - continue - if current_path != name.module_path: - current_path = name.module_path - - process(current_path, old_lines, new_lines) - if current_path is not None: - # None means take the source that is a normal param. - with open(current_path) as f: - source = f.read() - - new_lines = common.splitlines(common.source_to_unicode(source)) - old_lines = new_lines[:] - - nr, indent = name.line, name.column - line = new_lines[nr - 1] - new_lines[nr - 1] = line[:indent] + replace_str + \ - line[indent + len(name.name):] - process(current_path, old_lines, new_lines) - return dct - - -def extract(script, new_name): - """ The `args` / `kwargs` params are the same as in `api.Script`. - :param operation: The refactoring operation to execute. - :type operation: str - :type source: str - :return: list of changed lines/changed files - """ - new_lines = common.splitlines(common.source_to_unicode(script.source)) - old_lines = new_lines[:] - - user_stmt = script._parser.user_stmt() - - # TODO care for multiline extracts - dct = {} - if user_stmt: - pos = script._pos - line_index = pos[0] - 1 - arr, index = helpers.array_for_pos(user_stmt, pos) - if arr is not None: - start_pos = arr[index].start_pos - end_pos = arr[index].end_pos - - # take full line if the start line is different from end line - e = end_pos[1] if end_pos[0] == start_pos[0] else None - start_line = new_lines[start_pos[0] - 1] - text = start_line[start_pos[1]:e] - for l in range(start_pos[0], end_pos[0] - 1): - text += '\n' + l - if e is None: - end_line = new_lines[end_pos[0] - 1] - text += '\n' + end_line[:end_pos[1]] - - # remove code from new lines - t = text.lstrip() - del_start = start_pos[1] + len(text) - len(t) - - text = t.rstrip() - del_end = len(t) - len(text) - if e is None: - new_lines[end_pos[0] - 1] = end_line[end_pos[1] - del_end:] - e = len(start_line) - else: - e = e - del_end - start_line = start_line[:del_start] + new_name + start_line[e:] - new_lines[start_pos[0] - 1] = start_line - new_lines[start_pos[0]:end_pos[0] - 1] = [] - - # add parentheses in multiline case - open_brackets = ['(', '[', '{'] - close_brackets = [')', ']', '}'] - if '\n' in text and not (text[0] in open_brackets and text[-1] == - close_brackets[open_brackets.index(text[0])]): - text = '(%s)' % text - - # add new line before statement - indent = user_stmt.start_pos[1] - new = "%s%s = %s" % (' ' * indent, new_name, text) - new_lines.insert(line_index, new) - dct[script.path] = script.path, old_lines, new_lines - return Refactoring(dct) - - -def inline(script): - """ - :type script: api.Script - """ - new_lines = common.splitlines(common.source_to_unicode(script.source)) - - dct = {} - - definitions = script.goto_assignments() - with common.ignored(AssertionError): - assert len(definitions) == 1 - stmt = definitions[0]._definition - usages = script.usages() - inlines = [r for r in usages - if not stmt.start_pos <= (r.line, r.column) <= stmt.end_pos] - inlines = sorted(inlines, key=lambda x: (x.module_path, x.line, x.column), - reverse=True) - expression_list = stmt.expression_list() - # don't allow multiline refactorings for now. - assert stmt.start_pos[0] == stmt.end_pos[0] - index = stmt.start_pos[0] - 1 - - line = new_lines[index] - replace_str = line[expression_list[0].start_pos[1]:stmt.end_pos[1] + 1] - replace_str = replace_str.strip() - # tuples need parentheses - if expression_list and isinstance(expression_list[0], pr.Array): - arr = expression_list[0] - if replace_str[0] not in ['(', '[', '{'] and len(arr) > 1: - replace_str = '(%s)' % replace_str - - # if it's the only assignment, remove the statement - if len(stmt.get_defined_names()) == 1: - line = line[:stmt.start_pos[1]] + line[stmt.end_pos[1]:] - - dct = _rename(inlines, replace_str) - # remove the empty line - new_lines = dct[script.path][2] - if line.strip(): - new_lines[index] = line - else: - new_lines.pop(index) - - return Refactoring(dct) diff --git a/pythonFiles/preview/jedi/settings.py b/pythonFiles/preview/jedi/settings.py deleted file mode 100644 index 17cf9a0237b6..000000000000 --- a/pythonFiles/preview/jedi/settings.py +++ /dev/null @@ -1,233 +0,0 @@ -""" -This module contains variables with global |jedi| settings. To change the -behavior of |jedi|, change the variables defined in :mod:`jedi.settings`. - -Plugins should expose an interface so that the user can adjust the -configuration. - - -Example usage:: - - from jedi import settings - settings.case_insensitive_completion = True - - -Completion output -~~~~~~~~~~~~~~~~~ - -.. autodata:: case_insensitive_completion -.. autodata:: add_dot_after_module -.. autodata:: add_bracket_after_function -.. autodata:: no_completion_duplicates - - -Filesystem cache -~~~~~~~~~~~~~~~~ - -.. autodata:: cache_directory -.. autodata:: use_filesystem_cache - - -Parser -~~~~~~ - -.. autodata:: fast_parser - - -Dynamic stuff -~~~~~~~~~~~~~ - -.. autodata:: dynamic_array_additions -.. autodata:: dynamic_params -.. autodata:: dynamic_params_for_other_modules -.. autodata:: additional_dynamic_modules -.. autodata:: auto_import_modules - - -.. _settings-recursion: - -Recursions -~~~~~~~~~~ - -Recursion settings are important if you don't want extremly -recursive python code to go absolutely crazy. First of there is a -global limit :data:`max_executions`. This limit is important, to set -a maximum amount of time, the completion may use. - -The default values are based on experiments while completing the |jedi| library -itself (inception!). But I don't think there's any other Python library that -uses recursion in a similarly extreme way. These settings make the completion -definitely worse in some cases. But a completion should also be fast. - -.. autodata:: max_until_execution_unique -.. autodata:: max_function_recursion_level -.. autodata:: max_executions_without_builtins -.. autodata:: max_executions -.. autodata:: max_dynamic_params_depth -.. autodata:: scale_call_signatures - - -Caching -~~~~~~~ - -.. autodata:: star_import_cache_validity -.. autodata:: call_signatures_validity - - -""" -import os -import platform - -# ---------------- -# completion output settings -# ---------------- - -case_insensitive_completion = True -""" -The completion is by default case insensitive. -""" - -add_bracket_after_function = False -""" -Adds an opening bracket after a function, because that's normal behaviour. -Removed it again, because in VIM that is not very practical. -""" - -no_completion_duplicates = True -""" -If set, completions with the same name don't appear in the output anymore, -but are in the `same_name_completions` attribute. -""" - -# ---------------- -# Filesystem cache -# ---------------- - -use_filesystem_cache = True -""" -Use filesystem cache to save once parsed files with pickle. -""" - -if platform.system().lower() == 'windows': - _cache_directory = os.path.join(os.getenv('LOCALAPPDATA') or '~', 'Jedi', - 'Jedi') -elif platform.system().lower() == 'darwin': - _cache_directory = os.path.join('~', 'Library', 'Caches', 'Jedi') -else: - _cache_directory = os.path.join(os.getenv('XDG_CACHE_HOME') or '~/.cache', - 'jedi') -cache_directory = os.path.expanduser(_cache_directory) -""" -The path where all the caches can be found. - -On Linux, this defaults to ``~/.cache/jedi/``, on OS X to -``~/Library/Caches/Jedi/`` and on Windows to ``%APPDATA%\\Jedi\\Jedi\\``. -On Linux, if environment variable ``$XDG_CACHE_HOME`` is set, -``$XDG_CACHE_HOME/jedi`` is used instead of the default one. -""" - -# ---------------- -# parser -# ---------------- - -fast_parser = True -""" -Use the fast parser. This means that reparsing is only being done if -something has been changed e.g. to a function. If this happens, only the -function is being reparsed. -""" - -# ---------------- -# dynamic stuff -# ---------------- - -dynamic_array_additions = True -""" -check for `append`, etc. on arrays: [], {}, () as well as list/set calls. -""" - -dynamic_params = True -""" -A dynamic param completion, finds the callees of the function, which define -the params of a function. -""" - -dynamic_params_for_other_modules = True -""" -Do the same for other modules. -""" - -additional_dynamic_modules = [] -""" -Additional modules in which |jedi| checks if statements are to be found. This -is practical for IDEs, that want to administrate their modules themselves. -""" - -dynamic_flow_information = True -""" -Check for `isinstance` and other information to infer a type. -""" - -auto_import_modules = [ - 'hashlib', # setattr -] -""" -Modules that are not analyzed but imported, although they contain Python code. -This improves autocompletion for libraries that use ``setattr`` or -``globals()`` modifications a lot. -""" - -# ---------------- -# recursions -# ---------------- - -max_until_execution_unique = 50 -""" -This limit is probably the most important one, because if this limit is -exceeded, functions can only be one time executed. So new functions will be -executed, complex recursions with the same functions again and again, are -ignored. -""" - -max_function_recursion_level = 5 -""" -`max_function_recursion_level` is more about whether the recursions are -stopped in deepth or in width. The ratio beetween this and -`max_until_execution_unique` is important here. It stops a recursion (after -the number of function calls in the recursion), if it was already used -earlier. -""" - -max_executions_without_builtins = 200 -""" -.. todo:: Document this. -""" - -max_executions = 250 -""" -A maximum amount of time, the completion may use. -""" - -scale_call_signatures = 0.1 -""" -Because call_signatures is normally used on every single key hit, it has -to be faster than a normal completion. This is the factor that is used to -scale `max_executions` and `max_until_execution_unique`: -""" - -# ---------------- -# caching validity (time) -# ---------------- - -star_import_cache_validity = 60.0 -""" -In huge packages like numpy, checking all star imports on every completion -might be slow, therefore we do a star import caching, that lasts a certain -time span (in seconds). -""" - -call_signatures_validity = 3.0 -""" -Finding function calls might be slow (0.1-0.5s). This is not acceptible for -normal writing. Therefore cache it for a short time. -""" diff --git a/pythonFiles/preview/jedi/utils.py b/pythonFiles/preview/jedi/utils.py deleted file mode 100644 index 44f7c0d20e03..000000000000 --- a/pythonFiles/preview/jedi/utils.py +++ /dev/null @@ -1,131 +0,0 @@ -""" -Utilities for end-users. -""" - -from __future__ import absolute_import -import __main__ -from collections import namedtuple -import logging -import traceback -import re -import os -import sys - -from jedi import Interpreter -from jedi.api.helpers import get_on_completion_name -from jedi import common - - -READLINE_DEBUG = False - - -def setup_readline(namespace_module=__main__): - """ - Install Jedi completer to :mod:`readline`. - - This function setups :mod:`readline` to use Jedi in Python interactive - shell. If you want to use a custom ``PYTHONSTARTUP`` file (typically - ``$HOME/.pythonrc.py``), you can add this piece of code:: - - try: - from jedi.utils import setup_readline - setup_readline() - except ImportError: - # Fallback to the stdlib readline completer if it is installed. - # Taken from http://docs.python.org/2/library/rlcompleter.html - print("Jedi is not installed, falling back to readline") - try: - import readline - import rlcompleter - readline.parse_and_bind("tab: complete") - except ImportError: - print("Readline is not installed either. No tab completion is enabled.") - - This will fallback to the readline completer if Jedi is not installed. - The readline completer will only complete names in the global namespace, - so for example:: - - ran - - will complete to ``range`` - - with both Jedi and readline, but:: - - range(10).cou - - will show complete to ``range(10).count`` only with Jedi. - - You'll also need to add ``export PYTHONSTARTUP=$HOME/.pythonrc.py`` to - your shell profile (usually ``.bash_profile`` or ``.profile`` if you use - bash). - - """ - if READLINE_DEBUG: - logging.basicConfig( - filename='/tmp/jedi.log', - filemode='a', - level=logging.DEBUG - ) - - class JediRL(object): - def complete(self, text, state): - """ - This complete stuff is pretty weird, a generator would make - a lot more sense, but probably due to backwards compatibility - this is still the way how it works. - - The only important part is stuff in the ``state == 0`` flow, - everything else has been copied from the ``rlcompleter`` std. - library module. - """ - if state == 0: - sys.path.insert(0, os.getcwd()) - # Calling python doesn't have a path, so add to sys.path. - try: - logging.debug("Start REPL completion: " + repr(text)) - interpreter = Interpreter(text, [namespace_module.__dict__]) - - lines = common.splitlines(text) - position = (len(lines), len(lines[-1])) - name = get_on_completion_name(interpreter._get_module(), lines, position) - before = text[:len(text) - len(name)] - completions = interpreter.completions() - except: - logging.error("REPL Completion error:\n" + traceback.format_exc()) - raise - finally: - sys.path.pop(0) - - self.matches = [before + c.name_with_symbols for c in completions] - try: - return self.matches[state] - except IndexError: - return None - - try: - import readline - except ImportError: - print("Jedi: Module readline not available.") - else: - readline.set_completer(JediRL().complete) - readline.parse_and_bind("tab: complete") - # jedi itself does the case matching - readline.parse_and_bind("set completion-ignore-case on") - # because it's easier to hit the tab just once - readline.parse_and_bind("set show-all-if-unmodified") - readline.parse_and_bind("set show-all-if-ambiguous on") - # don't repeat all the things written in the readline all the time - readline.parse_and_bind("set completion-prefix-display-length 2") - # No delimiters, Jedi handles that. - readline.set_completer_delims('') - - -def version_info(): - """ - Returns a namedtuple of Jedi's version, similar to Python's - ``sys.version_info``. - """ - Version = namedtuple('Version', 'major, minor, micro') - from jedi import __version__ - tupl = re.findall('[a-z]+|\d+', __version__) - return Version(*[x if i == 3 else int(x) for i, x in enumerate(tupl)]) diff --git a/pythonFiles/release/jedi/__main__.py b/pythonFiles/release/jedi/__main__.py deleted file mode 100755 index f2ee0477695b..000000000000 --- a/pythonFiles/release/jedi/__main__.py +++ /dev/null @@ -1,48 +0,0 @@ -import sys -from os.path import join, dirname, abspath, isdir - - -def _start_linter(): - """ - This is a pre-alpha API. You're not supposed to use it at all, except for - testing. It will very likely change. - """ - import jedi - - if '--debug' in sys.argv: - jedi.set_debug_function() - - for path in sys.argv[2:]: - if path.startswith('--'): - continue - if isdir(path): - import fnmatch - import os - - paths = [] - for root, dirnames, filenames in os.walk(path): - for filename in fnmatch.filter(filenames, '*.py'): - paths.append(os.path.join(root, filename)) - else: - paths = [path] - - try: - for path in paths: - for error in jedi.Script(path=path)._analysis(): - print(error) - except Exception: - if '--pdb' in sys.argv: - import traceback - traceback.print_exc() - import pdb - pdb.post_mortem() - else: - raise - - -if len(sys.argv) == 2 and sys.argv[1] == 'repl': - # don't want to use __main__ only for repl yet, maybe we want to use it for - # something else. So just use the keyword ``repl`` for now. - print(join(dirname(abspath(__file__)), 'api', 'replstartup.py')) -elif len(sys.argv) > 1 and sys.argv[1] == 'linter': - _start_linter() diff --git a/pythonFiles/release/jedi/api/replstartup.py b/pythonFiles/release/jedi/api/replstartup.py deleted file mode 100755 index 5bfcc8ce889e..000000000000 --- a/pythonFiles/release/jedi/api/replstartup.py +++ /dev/null @@ -1,27 +0,0 @@ -""" -To use Jedi completion in Python interpreter, add the following in your shell -setup (e.g., ``.bashrc``):: - - export PYTHONSTARTUP="$(python -m jedi repl)" - -Then you will be able to use Jedi completer in your Python interpreter:: - - $ python - Python 2.7.2+ (default, Jul 20 2012, 22:15:08) - [GCC 4.6.1] on linux2 - Type "help", "copyright", "credits" or "license" for more information. - >>> import os - >>> os.path.join().split().in # doctest: +SKIP - os.path.join().split().index os.path.join().split().insert - -""" -import jedi.utils -from jedi import __version__ as __jedi_version__ - -print('REPL completion using Jedi %s' % __jedi_version__) -jedi.utils.setup_readline() - -del jedi - -# Note: try not to do many things here, as it will contaminate global -# namespace of the interpreter. diff --git a/pythonFiles/release/jedi/evaluate/compiled/fake/_functools.pym b/pythonFiles/release/jedi/evaluate/compiled/fake/_functools.pym deleted file mode 100755 index 909ef03fc3dd..000000000000 --- a/pythonFiles/release/jedi/evaluate/compiled/fake/_functools.pym +++ /dev/null @@ -1,9 +0,0 @@ -class partial(): - def __init__(self, func, *args, **keywords): - self.__func = func - self.__args = args - self.__keywords = keywords - - def __call__(self, *args, **kwargs): - # TODO should be **dict(self.__keywords, **kwargs) - return self.__func(*(self.__args + args), **self.__keywords) diff --git a/pythonFiles/release/jedi/evaluate/compiled/fake/_sqlite3.pym b/pythonFiles/release/jedi/evaluate/compiled/fake/_sqlite3.pym deleted file mode 100755 index 2151e652b401..000000000000 --- a/pythonFiles/release/jedi/evaluate/compiled/fake/_sqlite3.pym +++ /dev/null @@ -1,26 +0,0 @@ -def connect(database, timeout=None, isolation_level=None, detect_types=None, factory=None): - return Connection() - - -class Connection(): - def cursor(self): - return Cursor() - - -class Cursor(): - def cursor(self): - return Cursor() - - def fetchone(self): - return Row() - - def fetchmany(self, size=cursor.arraysize): - return [self.fetchone()] - - def fetchall(self): - return [self.fetchone()] - - -class Row(): - def keys(self): - return [''] diff --git a/pythonFiles/release/jedi/evaluate/compiled/fake/_sre.pym b/pythonFiles/release/jedi/evaluate/compiled/fake/_sre.pym deleted file mode 100755 index 217be5633982..000000000000 --- a/pythonFiles/release/jedi/evaluate/compiled/fake/_sre.pym +++ /dev/null @@ -1,99 +0,0 @@ -def compile(): - class SRE_Match(): - endpos = int() - lastgroup = int() - lastindex = int() - pos = int() - string = str() - regs = ((int(), int()),) - - def __init__(self, pattern): - self.re = pattern - - def start(self): - return int() - - def end(self): - return int() - - def span(self): - return int(), int() - - def expand(self): - return str() - - def group(self, nr): - return str() - - def groupdict(self): - return {str(): str()} - - def groups(self): - return (str(),) - - class SRE_Pattern(): - flags = int() - groupindex = {} - groups = int() - pattern = str() - - def findall(self, string, pos=None, endpos=None): - """ - findall(string[, pos[, endpos]]) --> list. - Return a list of all non-overlapping matches of pattern in string. - """ - return [str()] - - def finditer(self, string, pos=None, endpos=None): - """ - finditer(string[, pos[, endpos]]) --> iterator. - Return an iterator over all non-overlapping matches for the - RE pattern in string. For each match, the iterator returns a - match object. - """ - yield SRE_Match(self) - - def match(self, string, pos=None, endpos=None): - """ - match(string[, pos[, endpos]]) --> match object or None. - Matches zero or more characters at the beginning of the string - pattern - """ - return SRE_Match(self) - - def scanner(self, string, pos=None, endpos=None): - pass - - def search(self, string, pos=None, endpos=None): - """ - search(string[, pos[, endpos]]) --> match object or None. - Scan through string looking for a match, and return a corresponding - MatchObject instance. Return None if no position in the string matches. - """ - return SRE_Match(self) - - def split(self, string, maxsplit=0]): - """ - split(string[, maxsplit = 0]) --> list. - Split string by the occurrences of pattern. - """ - return [str()] - - def sub(self, repl, string, count=0): - """ - sub(repl, string[, count = 0]) --> newstring - Return the string obtained by replacing the leftmost non-overlapping - occurrences of pattern in string by the replacement repl. - """ - return str() - - def subn(self, repl, string, count=0): - """ - subn(repl, string[, count = 0]) --> (newstring, number of subs) - Return the tuple (new_string, number_of_subs_made) found by replacing - the leftmost non-overlapping occurrences of pattern with the - replacement repl. - """ - return (str(), int()) - - return SRE_Pattern() diff --git a/pythonFiles/release/jedi/evaluate/compiled/fake/datetime.pym b/pythonFiles/release/jedi/evaluate/compiled/fake/datetime.pym deleted file mode 100755 index 823ac5b7fd56..000000000000 --- a/pythonFiles/release/jedi/evaluate/compiled/fake/datetime.pym +++ /dev/null @@ -1,4 +0,0 @@ -class datetime(): - @staticmethod - def now(): - return datetime() diff --git a/pythonFiles/release/jedi/evaluate/compiled/fake/posix.pym b/pythonFiles/release/jedi/evaluate/compiled/fake/posix.pym deleted file mode 100755 index 4417f7cb0427..000000000000 --- a/pythonFiles/release/jedi/evaluate/compiled/fake/posix.pym +++ /dev/null @@ -1,5 +0,0 @@ -def getcwd(): - return '' - -def getcwdu(): - return '' diff --git a/pythonFiles/release/jedi/evaluate/jedi_typing.py b/pythonFiles/release/jedi/evaluate/jedi_typing.py deleted file mode 100644 index f48a567327eb..000000000000 --- a/pythonFiles/release/jedi/evaluate/jedi_typing.py +++ /dev/null @@ -1,100 +0,0 @@ -""" -This module is not intended to be used in jedi, rather it will be fed to the -jedi-parser to replace classes in the typing module -""" - -try: - from collections import abc -except ImportError: - # python 2 - import collections as abc - - -def factory(typing_name, indextypes): - class Iterable(abc.Iterable): - def __iter__(self): - while True: - yield indextypes[0]() - - class Iterator(Iterable, abc.Iterator): - def next(self): - """ needed for python 2 """ - return self.__next__() - - def __next__(self): - return indextypes[0]() - - class Sequence(abc.Sequence): - def __getitem__(self, index): - return indextypes[0]() - - class MutableSequence(Sequence, abc.MutableSequence): - pass - - class List(MutableSequence, list): - pass - - class Tuple(Sequence, tuple): - def __getitem__(self, index): - if indextypes[1] == Ellipsis: - # https://www.python.org/dev/peps/pep-0484/#the-typing-module - # Tuple[int, ...] means a tuple of ints of indetermined length - return indextypes[0]() - else: - return indextypes[index]() - - class AbstractSet(Iterable, abc.Set): - pass - - class MutableSet(AbstractSet, abc.MutableSet): - pass - - class KeysView(Iterable, abc.KeysView): - pass - - class ValuesView(abc.ValuesView): - def __iter__(self): - while True: - yield indextypes[1]() - - class ItemsView(abc.ItemsView): - def __iter__(self): - while True: - yield (indextypes[0](), indextypes[1]()) - - class Mapping(Iterable, abc.Mapping): - def __getitem__(self, item): - return indextypes[1]() - - def keys(self): - return KeysView() - - def values(self): - return ValuesView() - - def items(self): - return ItemsView() - - class MutableMapping(Mapping, abc.MutableMapping): - pass - - class Dict(MutableMapping, dict): - pass - - dct = { - "Sequence": Sequence, - "MutableSequence": MutableSequence, - "List": List, - "Iterable": Iterable, - "Iterator": Iterator, - "AbstractSet": AbstractSet, - "MutableSet": MutableSet, - "Mapping": Mapping, - "MutableMapping": MutableMapping, - "Tuple": Tuple, - "KeysView": KeysView, - "ItemsView": ItemsView, - "ValuesView": ValuesView, - "Dict": Dict, - } - return dct[typing_name] diff --git a/pythonFiles/release/jedi/evaluate/site.py b/pythonFiles/release/jedi/evaluate/site.py deleted file mode 100644 index bf884faefaaf..000000000000 --- a/pythonFiles/release/jedi/evaluate/site.py +++ /dev/null @@ -1,110 +0,0 @@ -"""An adapted copy of relevant site-packages functionality from Python stdlib. - -This file contains some functions related to handling site-packages in Python -with jedi-specific modifications: - -- the functions operate on sys_path argument rather than global sys.path - -- in .pth files "import ..." lines that allow execution of arbitrary code are - skipped to prevent code injection into jedi interpreter - -""" - -# Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, -# 2011, 2012, 2013, 2014, 2015 Python Software Foundation; All Rights Reserved - -from __future__ import print_function - -import sys -import os - - -def makepath(*paths): - dir = os.path.join(*paths) - try: - dir = os.path.abspath(dir) - except OSError: - pass - return dir, os.path.normcase(dir) - - -def _init_pathinfo(sys_path): - """Return a set containing all existing directory entries from sys_path""" - d = set() - for dir in sys_path: - try: - if os.path.isdir(dir): - dir, dircase = makepath(dir) - d.add(dircase) - except TypeError: - continue - return d - - -def addpackage(sys_path, sitedir, name, known_paths): - """Process a .pth file within the site-packages directory: - For each line in the file, either combine it with sitedir to a path - and add that to known_paths, or execute it if it starts with 'import '. - """ - if known_paths is None: - known_paths = _init_pathinfo(sys_path) - reset = 1 - else: - reset = 0 - fullname = os.path.join(sitedir, name) - try: - f = open(fullname, "r") - except OSError: - return - with f: - for n, line in enumerate(f): - if line.startswith("#"): - continue - try: - if line.startswith(("import ", "import\t")): - # Change by immerrr: don't evaluate import lines to prevent - # code injection into jedi through pth files. - # - # exec(line) - continue - line = line.rstrip() - dir, dircase = makepath(sitedir, line) - if not dircase in known_paths and os.path.exists(dir): - sys_path.append(dir) - known_paths.add(dircase) - except Exception: - print("Error processing line {:d} of {}:\n".format(n+1, fullname), - file=sys.stderr) - import traceback - for record in traceback.format_exception(*sys.exc_info()): - for line in record.splitlines(): - print(' '+line, file=sys.stderr) - print("\nRemainder of file ignored", file=sys.stderr) - break - if reset: - known_paths = None - return known_paths - - -def addsitedir(sys_path, sitedir, known_paths=None): - """Add 'sitedir' argument to sys_path if missing and handle .pth files in - 'sitedir'""" - if known_paths is None: - known_paths = _init_pathinfo(sys_path) - reset = 1 - else: - reset = 0 - sitedir, sitedircase = makepath(sitedir) - if not sitedircase in known_paths: - sys_path.append(sitedir) # Add path component - known_paths.add(sitedircase) - try: - names = os.listdir(sitedir) - except OSError: - return - names = [name for name in names if name.endswith(".pth")] - for name in sorted(names): - addpackage(sys_path, sitedir, name, known_paths) - if reset: - known_paths = None - return known_paths diff --git a/pythonFiles/release/jedi/parser/__init__.py b/pythonFiles/release/jedi/parser/__init__.py deleted file mode 100755 index da4e25580968..000000000000 --- a/pythonFiles/release/jedi/parser/__init__.py +++ /dev/null @@ -1,395 +0,0 @@ -""" -The ``Parser`` tries to convert the available Python code in an easy to read -format, something like an abstract syntax tree. The classes who represent this -tree, are sitting in the :mod:`jedi.parser.tree` module. - -The Python module ``tokenize`` is a very important part in the ``Parser``, -because it splits the code into different words (tokens). Sometimes it looks a -bit messy. Sorry for that! You might ask now: "Why didn't you use the ``ast`` -module for this? Well, ``ast`` does a very good job understanding proper Python -code, but fails to work as soon as there's a single line of broken code. - -There's one important optimization that needs to be known: Statements are not -being parsed completely. ``Statement`` is just a representation of the tokens -within the statement. This lowers memory usage and cpu time and reduces the -complexity of the ``Parser`` (there's another parser sitting inside -``Statement``, which produces ``Array`` and ``Call``). -""" -import os -import re - -from jedi.parser import tree as pt -from jedi.parser import tokenize -from jedi.parser import token -from jedi.parser.token import (DEDENT, INDENT, ENDMARKER, NEWLINE, NUMBER, - STRING, OP, ERRORTOKEN) -from jedi.parser.pgen2.pgen import generate_grammar -from jedi.parser.pgen2.parse import PgenParser - -OPERATOR_KEYWORDS = 'and', 'for', 'if', 'else', 'in', 'is', 'lambda', 'not', 'or' -# Not used yet. In the future I intend to add something like KeywordStatement -STATEMENT_KEYWORDS = 'assert', 'del', 'global', 'nonlocal', 'raise', \ - 'return', 'yield', 'pass', 'continue', 'break' - - -_loaded_grammars = {} - - -def load_grammar(file='grammar3.4'): - # For now we only support two different Python syntax versions: The latest - # Python 3 and Python 2. This may change. - if file.startswith('grammar3'): - file = 'grammar3.4' - else: - file = 'grammar2.7' - - global _loaded_grammars - path = os.path.join(os.path.dirname(__file__), file) + '.txt' - try: - return _loaded_grammars[path] - except KeyError: - return _loaded_grammars.setdefault(path, generate_grammar(path)) - - -class ErrorStatement(object): - def __init__(self, stack, next_token, position_modifier, next_start_pos): - self.stack = stack - self._position_modifier = position_modifier - self.next_token = next_token - self._next_start_pos = next_start_pos - - @property - def next_start_pos(self): - s = self._next_start_pos - return s[0] + self._position_modifier.line, s[1] - - @property - def first_pos(self): - first_type, nodes = self.stack[0] - return nodes[0].start_pos - - @property - def first_type(self): - first_type, nodes = self.stack[0] - return first_type - - -class ParserSyntaxError(object): - def __init__(self, message, position): - self.message = message - self.position = position - - -class Parser(object): - """ - This class is used to parse a Python file, it then divides them into a - class structure of different scopes. - - :param grammar: The grammar object of pgen2. Loaded by load_grammar. - :param source: The codebase for the parser. Must be unicode. - :param module_path: The path of the module in the file system, may be None. - :type module_path: str - :param top_module: Use this module as a parent instead of `self.module`. - """ - def __init__(self, grammar, source, module_path=None, tokenizer=None): - self._ast_mapping = { - 'expr_stmt': pt.ExprStmt, - 'classdef': pt.Class, - 'funcdef': pt.Function, - 'file_input': pt.Module, - 'import_name': pt.ImportName, - 'import_from': pt.ImportFrom, - 'break_stmt': pt.KeywordStatement, - 'continue_stmt': pt.KeywordStatement, - 'return_stmt': pt.ReturnStmt, - 'raise_stmt': pt.KeywordStatement, - 'yield_expr': pt.YieldExpr, - 'del_stmt': pt.KeywordStatement, - 'pass_stmt': pt.KeywordStatement, - 'global_stmt': pt.GlobalStmt, - 'nonlocal_stmt': pt.KeywordStatement, - 'assert_stmt': pt.AssertStmt, - 'if_stmt': pt.IfStmt, - 'with_stmt': pt.WithStmt, - 'for_stmt': pt.ForStmt, - 'while_stmt': pt.WhileStmt, - 'try_stmt': pt.TryStmt, - 'comp_for': pt.CompFor, - 'decorator': pt.Decorator, - 'lambdef': pt.Lambda, - 'old_lambdef': pt.Lambda, - 'lambdef_nocond': pt.Lambda, - } - - self.syntax_errors = [] - - self._global_names = [] - self._omit_dedent_list = [] - self._indent_counter = 0 - self._last_failed_start_pos = (0, 0) - - # TODO do print absolute import detection here. - #try: - # del python_grammar_no_print_statement.keywords["print"] - #except KeyError: - # pass # Doesn't exist in the Python 3 grammar. - - #if self.options["print_function"]: - # python_grammar = pygram.python_grammar_no_print_statement - #else: - self._used_names = {} - self._scope_names_stack = [{}] - self._error_statement_stacks = [] - - added_newline = False - # The Python grammar needs a newline at the end of each statement. - if not source.endswith('\n'): - source += '\n' - added_newline = True - - # For the fast parser. - self.position_modifier = pt.PositionModifier() - p = PgenParser(grammar, self.convert_node, self.convert_leaf, - self.error_recovery) - tokenizer = tokenizer or tokenize.source_tokens(source) - self.module = p.parse(self._tokenize(tokenizer)) - if self.module.type != 'file_input': - # If there's only one statement, we get back a non-module. That's - # not what we want, we want a module, so we add it here: - self.module = self.convert_node(grammar, - grammar.symbol2number['file_input'], - [self.module]) - - if added_newline: - self.remove_last_newline() - self.module.used_names = self._used_names - self.module.path = module_path - self.module.global_names = self._global_names - self.module.error_statement_stacks = self._error_statement_stacks - - def convert_node(self, grammar, type, children): - """ - Convert raw node information to a Node instance. - - This is passed to the parser driver which calls it whenever a reduction of a - grammar rule produces a new complete node, so that the tree is build - strictly bottom-up. - """ - symbol = grammar.number2symbol[type] - try: - new_node = self._ast_mapping[symbol](children) - except KeyError: - new_node = pt.Node(symbol, children) - - # We need to check raw_node always, because the same node can be - # returned by convert multiple times. - if symbol == 'global_stmt': - self._global_names += new_node.get_global_names() - elif isinstance(new_node, pt.Lambda): - new_node.names_dict = self._scope_names_stack.pop() - elif isinstance(new_node, (pt.ClassOrFunc, pt.Module)) \ - and symbol in ('funcdef', 'classdef', 'file_input'): - # scope_name_stack handling - scope_names = self._scope_names_stack.pop() - if isinstance(new_node, pt.ClassOrFunc): - n = new_node.name - scope_names[n.value].remove(n) - # Set the func name of the current node - arr = self._scope_names_stack[-1].setdefault(n.value, []) - arr.append(n) - new_node.names_dict = scope_names - elif isinstance(new_node, pt.CompFor): - # The name definitions of comprehenions shouldn't be part of the - # current scope. They are part of the comprehension scope. - for n in new_node.get_defined_names(): - self._scope_names_stack[-1][n.value].remove(n) - return new_node - - def convert_leaf(self, grammar, type, value, prefix, start_pos): - #print('leaf', value, pytree.type_repr(type)) - if type == tokenize.NAME: - if value in grammar.keywords: - if value in ('def', 'class', 'lambda'): - self._scope_names_stack.append({}) - - return pt.Keyword(self.position_modifier, value, start_pos, prefix) - else: - name = pt.Name(self.position_modifier, value, start_pos, prefix) - # Keep a listing of all used names - arr = self._used_names.setdefault(name.value, []) - arr.append(name) - arr = self._scope_names_stack[-1].setdefault(name.value, []) - arr.append(name) - return name - elif type == STRING: - return pt.String(self.position_modifier, value, start_pos, prefix) - elif type == NUMBER: - return pt.Number(self.position_modifier, value, start_pos, prefix) - elif type in (NEWLINE, ENDMARKER): - return pt.Whitespace(self.position_modifier, value, start_pos, prefix) - else: - return pt.Operator(self.position_modifier, value, start_pos, prefix) - - def error_recovery(self, grammar, stack, typ, value, start_pos, prefix, - add_token_callback): - """ - This parser is written in a dynamic way, meaning that this parser - allows using different grammars (even non-Python). However, error - recovery is purely written for Python. - """ - def current_suite(stack): - # For now just discard everything that is not a suite or - # file_input, if we detect an error. - for index, (dfa, state, (typ, nodes)) in reversed(list(enumerate(stack))): - # `suite` can sometimes be only simple_stmt, not stmt. - symbol = grammar.number2symbol[typ] - if symbol == 'file_input': - break - elif symbol == 'suite' and len(nodes) > 1: - # suites without an indent in them get discarded. - break - elif symbol == 'simple_stmt' and len(nodes) > 1: - # simple_stmt can just be turned into a Node, if there are - # enough statements. Ignore the rest after that. - break - return index, symbol, nodes - - index, symbol, nodes = current_suite(stack) - if symbol == 'simple_stmt': - index -= 2 - (_, _, (typ, suite_nodes)) = stack[index] - symbol = grammar.number2symbol[typ] - suite_nodes.append(pt.Node(symbol, list(nodes))) - # Remove - nodes[:] = [] - nodes = suite_nodes - stack[index] - - #print('err', token.tok_name[typ], repr(value), start_pos, len(stack), index) - self._stack_removal(grammar, stack, index + 1, value, start_pos) - if typ == INDENT: - # For every deleted INDENT we have to delete a DEDENT as well. - # Otherwise the parser will get into trouble and DEDENT too early. - self._omit_dedent_list.append(self._indent_counter) - - if value in ('import', 'from', 'class', 'def', 'try', 'while', 'return'): - # Those can always be new statements. - add_token_callback(typ, value, prefix, start_pos) - elif typ == DEDENT and symbol == 'suite': - # Close the current suite, with DEDENT. - # Note that this may cause some suites to not contain any - # statements at all. This is contrary to valid Python syntax. We - # keep incomplete suites in Jedi to be able to complete param names - # or `with ... as foo` names. If we want to use this parser for - # syntax checks, we have to check in a separate turn if suites - # contain statements or not. However, a second check is necessary - # anyway (compile.c does that for Python), because Python's grammar - # doesn't stop you from defining `continue` in a module, etc. - add_token_callback(typ, value, prefix, start_pos) - - def _stack_removal(self, grammar, stack, start_index, value, start_pos): - def clear_names(children): - for c in children: - try: - clear_names(c.children) - except AttributeError: - if isinstance(c, pt.Name): - try: - self._scope_names_stack[-1][c.value].remove(c) - self._used_names[c.value].remove(c) - except ValueError: - pass # This may happen with CompFor. - except KeyError: - pass # this seems to happen for large files - - for dfa, state, node in stack[start_index:]: - clear_names(children=node[1]) - - failed_stack = [] - found = False - for dfa, state, (typ, nodes) in stack[start_index:]: - if nodes: - found = True - if found: - symbol = grammar.number2symbol[typ] - failed_stack.append((symbol, nodes)) - if nodes and nodes[0] in ('def', 'class', 'lambda'): - self._scope_names_stack.pop() - if failed_stack: - err = ErrorStatement(failed_stack, value, self.position_modifier, start_pos) - self._error_statement_stacks.append(err) - - self._last_failed_start_pos = start_pos - - stack[start_index:] = [] - - def _tokenize(self, tokenizer): - for typ, value, start_pos, prefix in tokenizer: - #print(tokenize.tok_name[typ], repr(value), start_pos, repr(prefix)) - if typ == DEDENT: - # We need to count indents, because if we just omit any DEDENT, - # we might omit them in the wrong place. - o = self._omit_dedent_list - if o and o[-1] == self._indent_counter: - o.pop() - continue - - self._indent_counter -= 1 - elif typ == INDENT: - self._indent_counter += 1 - elif typ == ERRORTOKEN: - self._add_syntax_error('Strange token', start_pos) - continue - - if typ == OP: - typ = token.opmap[value] - yield typ, value, prefix, start_pos - - def _add_syntax_error(self, message, position): - self.syntax_errors.append(ParserSyntaxError(message, position)) - - def __repr__(self): - return "<%s: %s>" % (type(self).__name__, self.module) - - def remove_last_newline(self): - """ - In all of this we need to work with _start_pos, because if we worked - with start_pos, we would need to check the position_modifier as well - (which is accounted for in the start_pos property). - """ - endmarker = self.module.children[-1] - # The newline is either in the endmarker as a prefix or the previous - # leaf as a newline token. - if endmarker.prefix.endswith('\n'): - endmarker.prefix = endmarker.prefix[:-1] - last_line = re.sub('.*\n', '', endmarker.prefix) - endmarker._start_pos = endmarker._start_pos[0] - 1, len(last_line) - else: - try: - newline = endmarker.get_previous() - except IndexError: - return # This means that the parser is empty. - while True: - if newline.value == '': - # Must be a DEDENT, just continue. - try: - newline = newline.get_previous() - except IndexError: - # If there's a statement that fails to be parsed, there - # will be no previous leaf. So just ignore it. - break - elif newline.value != '\n': - # This may happen if error correction strikes and removes - # a whole statement including '\n'. - break - else: - newline.value = '' - if self._last_failed_start_pos > newline._start_pos: - # It may be the case that there was a syntax error in a - # function. In that case error correction removes the - # right newline. So we use the previously assigned - # _last_failed_start_pos variable to account for that. - endmarker._start_pos = self._last_failed_start_pos - else: - endmarker._start_pos = newline._start_pos - break diff --git a/pythonFiles/release/jedi/parser/fast.py b/pythonFiles/release/jedi/parser/fast.py deleted file mode 100755 index 35bb85556b22..000000000000 --- a/pythonFiles/release/jedi/parser/fast.py +++ /dev/null @@ -1,580 +0,0 @@ -""" -Basically a parser that is faster, because it tries to parse only parts and if -anything changes, it only reparses the changed parts. But because it's not -finished (and still not working as I want), I won't document it any further. -""" -import re -from itertools import chain - -from jedi._compatibility import use_metaclass -from jedi import settings -from jedi.parser import Parser -from jedi.parser import tree -from jedi import cache -from jedi import debug -from jedi.parser.tokenize import (source_tokens, NEWLINE, - ENDMARKER, INDENT, DEDENT) - -FLOWS = 'if', 'else', 'elif', 'while', 'with', 'try', 'except', 'finally', 'for' - - -class FastModule(tree.Module): - type = 'file_input' - - def __init__(self, module_path): - super(FastModule, self).__init__([]) - self.modules = [] - self.reset_caches() - self.names_dict = {} - self.path = module_path - - def reset_caches(self): - self.modules = [] - try: - del self._used_names # Remove the used names cache. - except AttributeError: - pass # It was never used. - - @property - @cache.underscore_memoization - def used_names(self): - return MergedNamesDict([m.used_names for m in self.modules]) - - @property - def global_names(self): - return [name for m in self.modules for name in m.global_names] - - @property - def error_statement_stacks(self): - return [e for m in self.modules for e in m.error_statement_stacks] - - def __repr__(self): - return "" % (type(self).__name__, self.name, - self.start_pos[0], self.end_pos[0]) - - # To avoid issues with with the `parser.Parser`, we need setters that do - # nothing, because if pickle comes along and sets those values. - @global_names.setter - def global_names(self, value): - pass - - @error_statement_stacks.setter - def error_statement_stacks(self, value): - pass - - @used_names.setter - def used_names(self, value): - pass - - -class MergedNamesDict(object): - def __init__(self, dicts): - self.dicts = dicts - - def __iter__(self): - return iter(set(key for dct in self.dicts for key in dct)) - - def __getitem__(self, value): - return list(chain.from_iterable(dct.get(value, []) for dct in self.dicts)) - - def items(self): - dct = {} - for d in self.dicts: - for key, values in d.items(): - try: - dct_values = dct[key] - dct_values += values - except KeyError: - dct[key] = list(values) - return dct.items() - - def values(self): - lst = [] - for dct in self.dicts: - lst += dct.values() - return lst - - -class CachedFastParser(type): - """ This is a metaclass for caching `FastParser`. """ - def __call__(self, grammar, source, module_path=None): - if not settings.fast_parser: - return Parser(grammar, source, module_path) - - pi = cache.parser_cache.get(module_path, None) - if pi is None or isinstance(pi.parser, Parser): - p = super(CachedFastParser, self).__call__(grammar, source, module_path) - else: - p = pi.parser # pi is a `cache.ParserCacheItem` - p.update(source) - return p - - -class ParserNode(object): - def __init__(self, fast_module, parser, source): - self._fast_module = fast_module - self.parent = None - self._node_children = [] - - self.source = source - self.hash = hash(source) - self.parser = parser - - try: - # With fast_parser we have either 1 subscope or only statements. - self._content_scope = parser.module.subscopes[0] - except IndexError: - self._content_scope = parser.module - else: - self._rewrite_last_newline() - - # We need to be able to reset the original children of a parser. - self._old_children = list(self._content_scope.children) - - def _rewrite_last_newline(self): - """ - The ENDMARKER can contain a newline in the prefix. However this prefix - really belongs to the function - respectively to the next function or - parser node. If we don't rewrite that newline, we end up with a newline - in the wrong position, i.d. at the end of the file instead of in the - middle. - """ - c = self._content_scope.children - if tree.is_node(c[-1], 'suite'): # In a simple_stmt there's no DEDENT. - end_marker = self.parser.module.children[-1] - # Set the DEDENT prefix instead of the ENDMARKER. - c[-1].children[-1].prefix = end_marker.prefix - end_marker.prefix = '' - - def __repr__(self): - module = self.parser.module - try: - return '<%s: %s-%s>' % (type(self).__name__, module.start_pos, module.end_pos) - except IndexError: - # There's no module yet. - return '<%s: empty>' % type(self).__name__ - - def reset_node(self): - """ - Removes changes that were applied in this class. - """ - self._node_children = [] - scope = self._content_scope - scope.children = list(self._old_children) - try: - # This works if it's a MergedNamesDict. - # We are correcting it, because the MergedNamesDicts are artificial - # and can change after closing a node. - scope.names_dict = scope.names_dict.dicts[0] - except AttributeError: - pass - - def close(self): - """ - Closes the current parser node. This means that after this no further - nodes should be added anymore. - """ - # We only need to replace the dict if multiple dictionaries are used: - if self._node_children: - dcts = [n.parser.module.names_dict for n in self._node_children] - # Need to insert the own node as well. - dcts.insert(0, self._content_scope.names_dict) - self._content_scope.names_dict = MergedNamesDict(dcts) - - def parent_until_indent(self, indent=None): - if (indent is None or self._indent >= indent) and self.parent is not None: - self.close() - return self.parent.parent_until_indent(indent) - return self - - @property - def _indent(self): - if not self.parent: - return 0 - - return self.parser.module.children[0].start_pos[1] - - def add_node(self, node, line_offset): - """Adding a node means adding a node that was already added earlier""" - # Changing the line offsets is very important, because if they don't - # fit, all the start_pos values will be wrong. - m = node.parser.module - node.parser.position_modifier.line = line_offset - self._fast_module.modules.append(m) - node.parent = self - - self._node_children.append(node) - - # Insert parser objects into current structure. We only need to set the - # parents and children in a good way. - scope = self._content_scope - for child in m.children: - child.parent = scope - scope.children.append(child) - - return node - - def all_sub_nodes(self): - """ - Returns all nodes including nested ones. - """ - for n in self._node_children: - yield n - for y in n.all_sub_nodes(): - yield y - - @cache.underscore_memoization # Should only happen once! - def remove_last_newline(self): - self.parser.remove_last_newline() - - -class FastParser(use_metaclass(CachedFastParser)): - _FLOWS_NEED_SPACE = 'if', 'elif', 'while', 'with', 'except', 'for' - _FLOWS_NEED_COLON = 'else', 'try', 'except', 'finally' - _keyword_re = re.compile('^[ \t]*(def |class |@|(?:%s)|(?:%s)\s*:)' - % ('|'.join(_FLOWS_NEED_SPACE), - '|'.join(_FLOWS_NEED_COLON))) - - def __init__(self, grammar, source, module_path=None): - # set values like `tree.Module`. - self._grammar = grammar - self.module_path = module_path - self._reset_caches() - self.update(source) - - def _reset_caches(self): - self.module = FastModule(self.module_path) - self.current_node = ParserNode(self.module, self, '') - - def update(self, source): - # For testing purposes: It is important that the number of parsers used - # can be minimized. With these variables we can test against that. - self.number_parsers_used = 0 - self.number_of_splits = 0 - self.number_of_misses = 0 - self.module.reset_caches() - try: - self._parse(source) - except: - # FastParser is cached, be careful with exceptions. - self._reset_caches() - raise - - def _split_parts(self, source): - """ - Split the source code into different parts. This makes it possible to - parse each part seperately and therefore cache parts of the file and - not everything. - """ - def gen_part(): - text = ''.join(current_lines) - del current_lines[:] - self.number_of_splits += 1 - return text - - def just_newlines(current_lines): - for line in current_lines: - line = line.lstrip('\t \n\r') - if line and line[0] != '#': - return False - return True - - # Split only new lines. Distinction between \r\n is the tokenizer's - # job. - # It seems like there's no problem with form feed characters here, - # because we're not counting lines. - self._lines = source.splitlines(True) - current_lines = [] - is_decorator = False - # Use -1, because that indent is always smaller than any other. - indent_list = [-1, 0] - new_indent = False - parentheses_level = 0 - flow_indent = None - previous_line = None - # All things within flows are simply being ignored. - for i, l in enumerate(self._lines): - # Handle backslash newline escaping. - if l.endswith('\\\n') or l.endswith('\\\r\n'): - if previous_line is not None: - previous_line += l - else: - previous_line = l - continue - if previous_line is not None: - l = previous_line + l - previous_line = None - - # check for dedents - s = l.lstrip('\t \n\r') - indent = len(l) - len(s) - if not s or s[0] == '#': - current_lines.append(l) # Just ignore comments and blank lines - continue - - if new_indent: - if indent > indent_list[-2]: - # Set the actual indent, not just the random old indent + 1. - indent_list[-1] = indent - new_indent = False - - while indent <= indent_list[-2]: # -> dedent - indent_list.pop() - # This automatically resets the flow_indent if there was a - # dedent or a flow just on one line (with one simple_stmt). - new_indent = False - if flow_indent is None and current_lines and not parentheses_level: - yield gen_part() - flow_indent = None - - # Check lines for functions/classes and split the code there. - if flow_indent is None: - m = self._keyword_re.match(l) - if m: - # Strip whitespace and colon from flows as a check. - if m.group(1).strip(' \t\r\n:') in FLOWS: - if not parentheses_level: - flow_indent = indent - else: - if not is_decorator and not just_newlines(current_lines): - yield gen_part() - is_decorator = '@' == m.group(1) - if not is_decorator: - parentheses_level = 0 - # The new indent needs to be higher - indent_list.append(indent + 1) - new_indent = True - elif is_decorator: - is_decorator = False - - parentheses_level = \ - max(0, (l.count('(') + l.count('[') + l.count('{') - - l.count(')') - l.count(']') - l.count('}'))) - - current_lines.append(l) - if current_lines: - yield gen_part() - - def _parse(self, source): - """ :type source: str """ - added_newline = False - if not source or source[-1] != '\n': - # To be compatible with Pythons grammar, we need a newline at the - # end. The parser would handle it, but since the fast parser abuses - # the normal parser in various ways, we need to care for this - # ourselves. - source += '\n' - added_newline = True - - next_line_offset = line_offset = 0 - start = 0 - nodes = list(self.current_node.all_sub_nodes()) - # Now we can reset the node, because we have all the old nodes. - self.current_node.reset_node() - last_end_line = 1 - - for code_part in self._split_parts(source): - next_line_offset += code_part.count('\n') - # If the last code part parsed isn't equal to the current end_pos, - # we know that the parser went further (`def` start in a - # docstring). So just parse the next part. - if line_offset + 1 == last_end_line: - self.current_node = self._get_node(code_part, source[start:], - line_offset, nodes) - else: - # Means that some lines where not fully parsed. Parse it now. - # This is a very rare case. Should only happens with very - # strange code bits. - self.number_of_misses += 1 - while last_end_line < next_line_offset + 1: - line_offset = last_end_line - 1 - # We could calculate the src in a more complicated way to - # make caching here possible as well. However, this is - # complicated and error-prone. Since this is not very often - # called - just ignore it. - src = ''.join(self._lines[line_offset:]) - self.current_node = self._get_node(code_part, src, - line_offset, nodes) - last_end_line = self.current_node.parser.module.end_pos[0] - - debug.dbg('While parsing %s, line %s slowed down the fast parser.', - self.module_path, line_offset + 1) - - line_offset = next_line_offset - start += len(code_part) - - last_end_line = self.current_node.parser.module.end_pos[0] - - if added_newline: - self.current_node.remove_last_newline() - - # Now that the for loop is finished, we still want to close all nodes. - self.current_node = self.current_node.parent_until_indent() - self.current_node.close() - - debug.dbg('Parsed %s, with %s parsers in %s splits.' - % (self.module_path, self.number_parsers_used, - self.number_of_splits)) - - def _get_node(self, source, parser_code, line_offset, nodes): - """ - Side effect: Alters the list of nodes. - """ - indent = len(source) - len(source.lstrip('\t ')) - self.current_node = self.current_node.parent_until_indent(indent) - - h = hash(source) - for index, node in enumerate(nodes): - if node.hash == h and node.source == source: - node.reset_node() - nodes.remove(node) - break - else: - tokenizer = FastTokenizer(parser_code) - self.number_parsers_used += 1 - p = Parser(self._grammar, parser_code, self.module_path, tokenizer=tokenizer) - - end = line_offset + p.module.end_pos[0] - used_lines = self._lines[line_offset:end - 1] - code_part_actually_used = ''.join(used_lines) - - node = ParserNode(self.module, p, code_part_actually_used) - - self.current_node.add_node(node, line_offset) - return node - - -class FastTokenizer(object): - """ - Breaks when certain conditions are met, i.e. a new function or class opens. - """ - def __init__(self, source): - self.source = source - self._gen = source_tokens(source) - self._closed = False - - # fast parser options - self.current = self.previous = NEWLINE, '', (0, 0) - self._in_flow = False - self._is_decorator = False - self._first_stmt = True - self._parentheses_level = 0 - self._indent_counter = 0 - self._flow_indent_counter = 0 - self._returned_endmarker = False - self._expect_indent = False - - def __iter__(self): - return self - - def next(self): - """ Python 2 Compatibility """ - return self.__next__() - - def __next__(self): - if self._closed: - return self._finish_dedents() - - typ, value, start_pos, prefix = current = next(self._gen) - if typ == ENDMARKER: - self._closed = True - self._returned_endmarker = True - return current - - self.previous = self.current - self.current = current - - if typ == INDENT: - self._indent_counter += 1 - if not self._expect_indent and not self._first_stmt and not self._in_flow: - # This does not mean that there is an actual flow, it means - # that the INDENT is syntactically wrong. - self._flow_indent_counter = self._indent_counter - 1 - self._in_flow = True - self._expect_indent = False - elif typ == DEDENT: - self._indent_counter -= 1 - if self._in_flow: - if self._indent_counter == self._flow_indent_counter: - self._in_flow = False - else: - self._closed = True - return current - - if value in ('def', 'class') and self._parentheses_level \ - and re.search(r'\n[ \t]*\Z', prefix): - # Account for the fact that an open parentheses before a function - # will reset the parentheses counter, but new lines before will - # still be ignored. So check the prefix. - - # TODO what about flow parentheses counter resets in the tokenizer? - self._parentheses_level = 0 - return self._close() - - # Parentheses ignore the indentation rules. The other three stand for - # new lines. - if self.previous[0] in (NEWLINE, INDENT, DEDENT) \ - and not self._parentheses_level and typ not in (INDENT, DEDENT): - if not self._in_flow: - if value in FLOWS: - self._flow_indent_counter = self._indent_counter - self._first_stmt = False - elif value in ('def', 'class', '@'): - # The values here are exactly the same check as in - # _split_parts, but this time with tokenize and therefore - # precise. - if not self._first_stmt and not self._is_decorator: - return self._close() - - self._is_decorator = '@' == value - if not self._is_decorator: - self._first_stmt = False - self._expect_indent = True - elif self._expect_indent: - return self._close() - else: - self._first_stmt = False - - if value in '([{' and value: - self._parentheses_level += 1 - elif value in ')]}' and value: - # Ignore closing parentheses, because they are all - # irrelevant for the indentation. - self._parentheses_level = max(self._parentheses_level - 1, 0) - return current - - def _close(self): - if self._first_stmt: - # Continue like nothing has happened, because we want to enter - # the first class/function. - if self.current[1] != '@': - self._first_stmt = False - return self.current - else: - self._closed = True - return self._finish_dedents() - - def _finish_dedents(self): - if self._indent_counter: - self._indent_counter -= 1 - return DEDENT, '', self.current[2], '' - elif not self._returned_endmarker: - self._returned_endmarker = True - return ENDMARKER, '', self.current[2], self._get_prefix() - else: - raise StopIteration - - def _get_prefix(self): - """ - We're using the current prefix for the endmarker to not loose any - information. However we care about "lost" lines. The prefix of the - current line (indent) will always be included in the current line. - """ - cur = self.current - while cur[0] == DEDENT: - cur = next(self._gen) - prefix = cur[3] - - # \Z for the end of the string. $ is bugged, because it has the - # same behavior with or without re.MULTILINE. - return re.sub(r'[^\n]+\Z', '', prefix) diff --git a/pythonFiles/release/jedi/parser/pgen2/__init__.py b/pythonFiles/release/jedi/parser/pgen2/__init__.py deleted file mode 100755 index 1ddae5fea9f7..000000000000 --- a/pythonFiles/release/jedi/parser/pgen2/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -# Modifications: -# Copyright 2006 Google, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. -# Copyright 2014 David Halter. Integration into Jedi. -# Modifications are dual-licensed: MIT and PSF. diff --git a/pythonFiles/release/jedi/parser/pgen2/grammar.py b/pythonFiles/release/jedi/parser/pgen2/grammar.py deleted file mode 100755 index 414c0dbe9f01..000000000000 --- a/pythonFiles/release/jedi/parser/pgen2/grammar.py +++ /dev/null @@ -1,125 +0,0 @@ -# Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -# Modifications: -# Copyright 2014 David Halter. Integration into Jedi. -# Modifications are dual-licensed: MIT and PSF. - -"""This module defines the data structures used to represent a grammar. - -These are a bit arcane because they are derived from the data -structures used by Python's 'pgen' parser generator. - -There's also a table here mapping operators to their names in the -token module; the Python tokenize module reports all operators as the -fallback token code OP, but the parser needs the actual token code. - -""" - -# Python imports -import pickle - - -class Grammar(object): - """Pgen parsing tables conversion class. - - Once initialized, this class supplies the grammar tables for the - parsing engine implemented by parse.py. The parsing engine - accesses the instance variables directly. The class here does not - provide initialization of the tables; several subclasses exist to - do this (see the conv and pgen modules). - - The load() method reads the tables from a pickle file, which is - much faster than the other ways offered by subclasses. The pickle - file is written by calling dump() (after loading the grammar - tables using a subclass). The report() method prints a readable - representation of the tables to stdout, for debugging. - - The instance variables are as follows: - - symbol2number -- a dict mapping symbol names to numbers. Symbol - numbers are always 256 or higher, to distinguish - them from token numbers, which are between 0 and - 255 (inclusive). - - number2symbol -- a dict mapping numbers to symbol names; - these two are each other's inverse. - - states -- a list of DFAs, where each DFA is a list of - states, each state is a list of arcs, and each - arc is a (i, j) pair where i is a label and j is - a state number. The DFA number is the index into - this list. (This name is slightly confusing.) - Final states are represented by a special arc of - the form (0, j) where j is its own state number. - - dfas -- a dict mapping symbol numbers to (DFA, first) - pairs, where DFA is an item from the states list - above, and first is a set of tokens that can - begin this grammar rule (represented by a dict - whose values are always 1). - - labels -- a list of (x, y) pairs where x is either a token - number or a symbol number, and y is either None - or a string; the strings are keywords. The label - number is the index in this list; label numbers - are used to mark state transitions (arcs) in the - DFAs. - - start -- the number of the grammar's start symbol. - - keywords -- a dict mapping keyword strings to arc labels. - - tokens -- a dict mapping token numbers to arc labels. - - """ - - def __init__(self): - self.symbol2number = {} - self.number2symbol = {} - self.states = [] - self.dfas = {} - self.labels = [(0, "EMPTY")] - self.keywords = {} - self.tokens = {} - self.symbol2label = {} - self.start = 256 - - def dump(self, filename): - """Dump the grammar tables to a pickle file.""" - with open(filename, "wb") as f: - pickle.dump(self.__dict__, f, 2) - - def load(self, filename): - """Load the grammar tables from a pickle file.""" - with open(filename, "rb") as f: - d = pickle.load(f) - self.__dict__.update(d) - - def copy(self): - """ - Copy the grammar. - """ - new = self.__class__() - for dict_attr in ("symbol2number", "number2symbol", "dfas", "keywords", - "tokens", "symbol2label"): - setattr(new, dict_attr, getattr(self, dict_attr).copy()) - new.labels = self.labels[:] - new.states = self.states[:] - new.start = self.start - return new - - def report(self): - """Dump the grammar tables to standard output, for debugging.""" - from pprint import pprint - print("s2n") - pprint(self.symbol2number) - print("n2s") - pprint(self.number2symbol) - print("states") - pprint(self.states) - print("dfas") - pprint(self.dfas) - print("labels") - pprint(self.labels) - print("start", self.start) diff --git a/pythonFiles/release/jedi/parser/pgen2/parse.py b/pythonFiles/release/jedi/parser/pgen2/parse.py deleted file mode 100755 index c8ba70d356be..000000000000 --- a/pythonFiles/release/jedi/parser/pgen2/parse.py +++ /dev/null @@ -1,205 +0,0 @@ -# Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -# Modifications: -# Copyright 2014 David Halter. Integration into Jedi. -# Modifications are dual-licensed: MIT and PSF. - -""" -Parser engine for the grammar tables generated by pgen. - -The grammar table must be loaded first. - -See Parser/parser.c in the Python distribution for additional info on -how this parsing engine works. -""" - -# Local imports -from jedi.parser import tokenize - - -class ParseError(Exception): - """Exception to signal the parser is stuck.""" - - def __init__(self, msg, type, value, start_pos): - Exception.__init__(self, "%s: type=%r, value=%r, start_pos=%r" % - (msg, tokenize.tok_name[type], value, start_pos)) - self.msg = msg - self.type = type - self.value = value - self.start_pos = start_pos - - -class PgenParser(object): - """Parser engine. - - The proper usage sequence is: - - p = Parser(grammar, [converter]) # create instance - p.setup([start]) # prepare for parsing - : - if p.addtoken(...): # parse a token; may raise ParseError - break - root = p.rootnode # root of abstract syntax tree - - A Parser instance may be reused by calling setup() repeatedly. - - A Parser instance contains state pertaining to the current token - sequence, and should not be used concurrently by different threads - to parse separate token sequences. - - See driver.py for how to get input tokens by tokenizing a file or - string. - - Parsing is complete when addtoken() returns True; the root of the - abstract syntax tree can then be retrieved from the rootnode - instance variable. When a syntax error occurs, addtoken() raises - the ParseError exception. There is no error recovery; the parser - cannot be used after a syntax error was reported (but it can be - reinitialized by calling setup()). - - """ - - def __init__(self, grammar, convert_node, convert_leaf, error_recovery): - """Constructor. - - The grammar argument is a grammar.Grammar instance; see the - grammar module for more information. - - The parser is not ready yet for parsing; you must call the - setup() method to get it started. - - The optional convert argument is a function mapping concrete - syntax tree nodes to abstract syntax tree nodes. If not - given, no conversion is done and the syntax tree produced is - the concrete syntax tree. If given, it must be a function of - two arguments, the first being the grammar (a grammar.Grammar - instance), and the second being the concrete syntax tree node - to be converted. The syntax tree is converted from the bottom - up. - - A concrete syntax tree node is a (type, nodes) tuple, where - type is the node type (a token or symbol number) and nodes - is a list of children for symbols, and None for tokens. - - An abstract syntax tree node may be anything; this is entirely - up to the converter function. - - """ - self.grammar = grammar - self.convert_node = convert_node - self.convert_leaf = convert_leaf - - # Prepare for parsing. - start = self.grammar.start - # Each stack entry is a tuple: (dfa, state, node). - # A node is a tuple: (type, children), - # where children is a list of nodes or None - newnode = (start, []) - stackentry = (self.grammar.dfas[start], 0, newnode) - self.stack = [stackentry] - self.rootnode = None - self.error_recovery = error_recovery - - def parse(self, tokenizer): - for type, value, prefix, start_pos in tokenizer: - if self.addtoken(type, value, prefix, start_pos): - break - else: - # We never broke out -- EOF is too soon -- Unfinished statement. - self.error_recovery(self.grammar, self.stack, type, value, - start_pos, prefix, self.addtoken) - # Add the ENDMARKER again. - if not self.addtoken(type, value, prefix, start_pos): - raise ParseError("incomplete input", type, value, start_pos) - return self.rootnode - - def addtoken(self, type, value, prefix, start_pos): - """Add a token; return True if this is the end of the program.""" - # Map from token to label - if type == tokenize.NAME: - # Check for reserved words (keywords) - try: - ilabel = self.grammar.keywords[value] - except KeyError: - ilabel = self.grammar.tokens[type] - else: - ilabel = self.grammar.tokens[type] - - # Loop until the token is shifted; may raise exceptions - while True: - dfa, state, node = self.stack[-1] - states, first = dfa - arcs = states[state] - # Look for a state with this label - for i, newstate in arcs: - t, v = self.grammar.labels[i] - if ilabel == i: - # Look it up in the list of labels - assert t < 256 - # Shift a token; we're done with it - self.shift(type, value, newstate, prefix, start_pos) - # Pop while we are in an accept-only state - state = newstate - while states[state] == [(0, state)]: - self.pop() - if not self.stack: - # Done parsing! - return True - dfa, state, node = self.stack[-1] - states, first = dfa - # Done with this token - return False - elif t >= 256: - # See if it's a symbol and if we're in its first set - itsdfa = self.grammar.dfas[t] - itsstates, itsfirst = itsdfa - if ilabel in itsfirst: - # Push a symbol - self.push(t, itsdfa, newstate) - break # To continue the outer while loop - else: - if (0, state) in arcs: - # An accepting state, pop it and try something else - self.pop() - if not self.stack: - # Done parsing, but another token is input - raise ParseError("too much input", type, value, start_pos) - else: - self.error_recovery(self.grammar, self.stack, type, - value, start_pos, prefix, self.addtoken) - break - - def shift(self, type, value, newstate, prefix, start_pos): - """Shift a token. (Internal)""" - dfa, state, node = self.stack[-1] - newnode = self.convert_leaf(self.grammar, type, value, prefix, start_pos) - node[-1].append(newnode) - self.stack[-1] = (dfa, newstate, node) - - def push(self, type, newdfa, newstate): - """Push a nonterminal. (Internal)""" - dfa, state, node = self.stack[-1] - newnode = (type, []) - self.stack[-1] = (dfa, newstate, node) - self.stack.append((newdfa, 0, newnode)) - - def pop(self): - """Pop a nonterminal. (Internal)""" - popdfa, popstate, (type, children) = self.stack.pop() - # If there's exactly one child, return that child instead of creating a - # new node. We still create expr_stmt and file_input though, because a - # lot of Jedi depends on its logic. - if len(children) == 1: - newnode = children[0] - else: - newnode = self.convert_node(self.grammar, type, children) - - try: - # Equal to: - # dfa, state, node = self.stack[-1] - # symbol, children = node - self.stack[-1][2][1].append(newnode) - except IndexError: - # Stack is empty, set the rootnode. - self.rootnode = newnode diff --git a/pythonFiles/release/jedi/parser/token.py b/pythonFiles/release/jedi/parser/token.py deleted file mode 100755 index e9ab3a6272ff..000000000000 --- a/pythonFiles/release/jedi/parser/token.py +++ /dev/null @@ -1,84 +0,0 @@ -from __future__ import absolute_import - -from jedi._compatibility import is_py3 -from token import * - - -COMMENT = N_TOKENS -tok_name[COMMENT] = 'COMMENT' -N_TOKENS += 1 - -NL = N_TOKENS -tok_name[NL] = 'NL' -N_TOKENS += 1 - -if is_py3: - BACKQUOTE = N_TOKENS - tok_name[BACKQUOTE] = 'BACKQUOTE' - N_TOKENS += 1 -else: - RARROW = N_TOKENS - tok_name[RARROW] = 'RARROW' - N_TOKENS += 1 - ELLIPSIS = N_TOKENS - tok_name[ELLIPSIS] = 'ELLIPSIS' - N_TOKENS += 1 - - - -# Map from operator to number (since tokenize doesn't do this) - -opmap_raw = """\ -( LPAR -) RPAR -[ LSQB -] RSQB -: COLON -, COMMA -; SEMI -+ PLUS -- MINUS -* STAR -/ SLASH -| VBAR -& AMPER -< LESS -> GREATER -= EQUAL -. DOT -% PERCENT -` BACKQUOTE -{ LBRACE -} RBRACE -@ AT -== EQEQUAL -!= NOTEQUAL -<> NOTEQUAL -<= LESSEQUAL ->= GREATEREQUAL -~ TILDE -^ CIRCUMFLEX -<< LEFTSHIFT ->> RIGHTSHIFT -** DOUBLESTAR -+= PLUSEQUAL --= MINEQUAL -*= STAREQUAL -/= SLASHEQUAL -%= PERCENTEQUAL -&= AMPEREQUAL -|= VBAREQUAL -^= CIRCUMFLEXEQUAL -<<= LEFTSHIFTEQUAL ->>= RIGHTSHIFTEQUAL -**= DOUBLESTAREQUAL -// DOUBLESLASH -//= DOUBLESLASHEQUAL --> RARROW -... ELLIPSIS -""" - -opmap = {} -for line in opmap_raw.splitlines(): - op, name = line.split() - opmap[op] = globals()[name] diff --git a/pythonFiles/release/jedi/parser/tokenize.py b/pythonFiles/release/jedi/parser/tokenize.py deleted file mode 100755 index b38490468349..000000000000 --- a/pythonFiles/release/jedi/parser/tokenize.py +++ /dev/null @@ -1,290 +0,0 @@ -# -*- coding: utf-8 -*- -""" -This tokenizer has been copied from the ``tokenize.py`` standard library -tokenizer. The reason was simple: The standard library tokenizer fails -if the indentation is not right. The fast parser of jedi however requires -"wrong" indentation. - -Basically this is a stripped down version of the standard library module, so -you can read the documentation there. Additionally we included some speed and -memory optimizations here. -""" -from __future__ import absolute_import - -import string -import re -from io import StringIO -from jedi.parser.token import (tok_name, N_TOKENS, ENDMARKER, STRING, NUMBER, - NAME, OP, ERRORTOKEN, NEWLINE, INDENT, DEDENT) -from jedi._compatibility import is_py3 - - -cookie_re = re.compile("coding[:=]\s*([-\w.]+)") - - -if is_py3: - # Python 3 has str.isidentifier() to check if a char is a valid identifier - is_identifier = str.isidentifier -else: - namechars = string.ascii_letters + '_' - is_identifier = lambda s: s in namechars - - -COMMENT = N_TOKENS -tok_name[COMMENT] = 'COMMENT' - - -def group(*choices): - return '(' + '|'.join(choices) + ')' - - -def maybe(*choices): - return group(*choices) + '?' - - -# Note: we use unicode matching for names ("\w") but ascii matching for -# number literals. -whitespace = r'[ \f\t]*' -comment = r'#[^\r\n]*' -name = r'\w+' - -hex_number = r'0[xX][0-9a-fA-F]+' -bin_number = r'0[bB][01]+' -oct_number = r'0[oO][0-7]+' -dec_number = r'(?:0+|[1-9][0-9]*)' -int_number = group(hex_number, bin_number, oct_number, dec_number) -exponent = r'[eE][-+]?[0-9]+' -point_float = group(r'[0-9]+\.[0-9]*', r'\.[0-9]+') + maybe(exponent) -Expfloat = r'[0-9]+' + exponent -float_number = group(point_float, Expfloat) -imag_number = group(r'[0-9]+[jJ]', float_number + r'[jJ]') -number = group(imag_number, float_number, int_number) - -# Tail end of ' string. -single = r"[^'\\]*(?:\\.[^'\\]*)*'" -# Tail end of " string. -double = r'[^"\\]*(?:\\.[^"\\]*)*"' -# Tail end of ''' string. -single3 = r"[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''" -# Tail end of """ string. -double3 = r'[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""' -triple = group("[uUbB]?[rR]?'''", '[uUbB]?[rR]?"""') -# Single-line ' or " string. - -# Because of leftmost-then-longest match semantics, be sure to put the -# longest operators first (e.g., if = came before ==, == would get -# recognized as two instances of =). -operator = group(r"\*\*=?", r">>=?", r"<<=?", r"!=", - r"//=?", r"->", - r"[+\-*/%&|^=<>]=?", - r"~") - -bracket = '[][(){}]' -special = group(r'\r?\n', r'\.\.\.', r'[:;.,@]') -funny = group(operator, bracket, special) - -# First (or only) line of ' or " string. -cont_str = group(r"[bBuU]?[rR]?'[^\n'\\]*(?:\\.[^\n'\\]*)*" + - group("'", r'\\\r?\n'), - r'[bBuU]?[rR]?"[^\n"\\]*(?:\\.[^\n"\\]*)*' + - group('"', r'\\\r?\n')) -pseudo_extras = group(r'\\\r?\n', comment, triple) -pseudo_token = group(whitespace) + \ - group(pseudo_extras, number, funny, cont_str, name) - - -def _compile(expr): - return re.compile(expr, re.UNICODE) - - -pseudoprog, single3prog, double3prog = map( - _compile, (pseudo_token, single3, double3)) - -endprogs = {"'": _compile(single), '"': _compile(double), - "'''": single3prog, '"""': double3prog, - "r'''": single3prog, 'r"""': double3prog, - "b'''": single3prog, 'b"""': double3prog, - "u'''": single3prog, 'u"""': double3prog, - "R'''": single3prog, 'R"""': double3prog, - "B'''": single3prog, 'B"""': double3prog, - "U'''": single3prog, 'U"""': double3prog, - "br'''": single3prog, 'br"""': double3prog, - "bR'''": single3prog, 'bR"""': double3prog, - "Br'''": single3prog, 'Br"""': double3prog, - "BR'''": single3prog, 'BR"""': double3prog, - "ur'''": single3prog, 'ur"""': double3prog, - "uR'''": single3prog, 'uR"""': double3prog, - "Ur'''": single3prog, 'Ur"""': double3prog, - "UR'''": single3prog, 'UR"""': double3prog, - 'r': None, 'R': None, 'b': None, 'B': None} - -triple_quoted = {} -for t in ("'''", '"""', - "r'''", 'r"""', "R'''", 'R"""', - "b'''", 'b"""', "B'''", 'B"""', - "u'''", 'u"""', "U'''", 'U"""', - "br'''", 'br"""', "Br'''", 'Br"""', - "bR'''", 'bR"""', "BR'''", 'BR"""', - "ur'''", 'ur"""', "Ur'''", 'Ur"""', - "uR'''", 'uR"""', "UR'''", 'UR"""'): - triple_quoted[t] = t -single_quoted = {} -for t in ("'", '"', - "r'", 'r"', "R'", 'R"', - "b'", 'b"', "B'", 'B"', - "u'", 'u"', "U'", 'U"', - "br'", 'br"', "Br'", 'Br"', - "bR'", 'bR"', "BR'", 'BR"', - "ur'", 'ur"', "Ur'", 'Ur"', - "uR'", 'uR"', "UR'", 'UR"'): - single_quoted[t] = t - -del _compile - -tabsize = 8 - -ALWAYS_BREAK_TOKENS = (';', 'import', 'from', 'class', 'def', 'try', 'except', - 'finally', 'while', 'return') - - -def source_tokens(source): - """Generate tokens from a the source code (string).""" - source = source + '\n' # end with \n, because the parser needs it - readline = StringIO(source).readline - return generate_tokens(readline) - - -def generate_tokens(readline): - """ - A heavily modified Python standard library tokenizer. - - Additionally to the default information, yields also the prefix of each - token. This idea comes from lib2to3. The prefix contains all information - that is irrelevant for the parser like newlines in parentheses or comments. - """ - paren_level = 0 # count parentheses - indents = [0] - lnum = 0 - numchars = '0123456789' - contstr = '' - contline = None - # We start with a newline. This makes indent at the first position - # possible. It's not valid Python, but still better than an INDENT in the - # second line (and not in the first). This makes quite a few things in - # Jedi's fast parser possible. - new_line = True - prefix = '' # Should never be required, but here for safety - additional_prefix = '' - while True: # loop over lines in stream - line = readline() # readline returns empty when finished. See StringIO - if not line: - if contstr: - yield ERRORTOKEN, contstr, contstr_start, prefix - break - - lnum += 1 - pos, max = 0, len(line) - - if contstr: # continued string - endmatch = endprog.match(line) - if endmatch: - pos = endmatch.end(0) - yield STRING, contstr + line[:pos], contstr_start, prefix - contstr = '' - contline = None - else: - contstr = contstr + line - contline = contline + line - continue - - while pos < max: - pseudomatch = pseudoprog.match(line, pos) - if not pseudomatch: # scan for tokens - txt = line[pos] - if line[pos] in '"\'': - # If a literal starts but doesn't end the whole rest of the - # line is an error token. - txt = line[pos:] - yield ERRORTOKEN, txt, (lnum, pos), prefix - pos += 1 - continue - - prefix = additional_prefix + pseudomatch.group(1) - additional_prefix = '' - start, pos = pseudomatch.span(2) - spos = (lnum, start) - token, initial = line[start:pos], line[start] - - if new_line and initial not in '\r\n#': - new_line = False - if paren_level == 0: - if start > indents[-1]: - yield INDENT, '', spos, '' - indents.append(start) - while start < indents[-1]: - yield DEDENT, '', spos, '' - indents.pop() - - if (initial in numchars or # ordinary number - (initial == '.' and token != '.' and token != '...')): - yield NUMBER, token, spos, prefix - elif initial in '\r\n': - if not new_line and paren_level == 0: - yield NEWLINE, token, spos, prefix - else: - additional_prefix = prefix + token - new_line = True - elif initial == '#': # Comments - assert not token.endswith("\n") - additional_prefix = prefix + token - elif token in triple_quoted: - endprog = endprogs[token] - endmatch = endprog.match(line, pos) - if endmatch: # all on one line - pos = endmatch.end(0) - token = line[start:pos] - yield STRING, token, spos, prefix - else: - contstr_start = (lnum, start) # multiple lines - contstr = line[start:] - contline = line - break - elif initial in single_quoted or \ - token[:2] in single_quoted or \ - token[:3] in single_quoted: - if token[-1] == '\n': # continued string - contstr_start = lnum, start - endprog = (endprogs.get(initial) or endprogs.get(token[1]) - or endprogs.get(token[2])) - contstr = line[start:] - contline = line - break - else: # ordinary string - yield STRING, token, spos, prefix - elif is_identifier(initial): # ordinary name - if token in ALWAYS_BREAK_TOKENS: - paren_level = 0 - while True: - indent = indents.pop() - if indent > start: - yield DEDENT, '', spos, '' - else: - indents.append(indent) - break - yield NAME, token, spos, prefix - elif initial == '\\' and line[start:] in ('\\\n', '\\\r\n'): # continued stmt - additional_prefix += prefix + line[start:] - break - else: - if token in '([{': - paren_level += 1 - elif token in ')]}': - paren_level -= 1 - yield OP, token, spos, prefix - - end_pos = (lnum, max - 1) - # As the last position we just take the maximally possible position. We - # remove -1 for the last new line. - for indent in indents[1:]: - yield DEDENT, '', end_pos, '' - yield ENDMARKER, '', end_pos, prefix diff --git a/pythonFiles/release/jedi/parser/tree.py b/pythonFiles/release/jedi/parser/tree.py deleted file mode 100755 index 619067e1905f..000000000000 --- a/pythonFiles/release/jedi/parser/tree.py +++ /dev/null @@ -1,1222 +0,0 @@ -""" -If you know what an abstract syntax tree (AST) is, you'll see that this module -is pretty much that. The classes represent syntax elements like functions and -imports. - -This is the "business logic" part of the parser. There's a lot of logic here -that makes it easier for Jedi (and other libraries to deal with a Python syntax -tree. - -By using `get_code` on a module, you can get back the 1-to-1 representation of -the input given to the parser. This is important if you are using refactoring. - -The easiest way to play with this module is to use :class:`parsing.Parser`. -:attr:`parsing.Parser.module` holds an instance of :class:`Module`: - ->>> from jedi._compatibility import u ->>> from jedi.parser import Parser, load_grammar ->>> parser = Parser(load_grammar(), u('import os'), 'example.py') ->>> submodule = parser.module ->>> submodule - - -Any subclasses of :class:`Scope`, including :class:`Module` has an attribute -:attr:`imports `: - ->>> submodule.imports -[] - -See also :attr:`Scope.subscopes` and :attr:`Scope.statements`. -""" -import os -import re -from inspect import cleandoc -from itertools import chain -import textwrap - -from jedi._compatibility import (Python3Method, encoding, is_py3, utf8_repr, - literal_eval, use_metaclass, unicode) -from jedi import cache - - -def is_node(node, *symbol_names): - try: - type = node.type - except AttributeError: - return False - else: - return type in symbol_names - - -class PositionModifier(object): - """A start_pos modifier for the fast parser.""" - def __init__(self): - self.line = 0 - - -zero_position_modifier = PositionModifier() - - -class DocstringMixin(object): - __slots__ = () - - @property - def raw_doc(self): - """ Returns a cleaned version of the docstring token. """ - if isinstance(self, Module): - node = self.children[0] - elif isinstance(self, ClassOrFunc): - node = self.children[self.children.index(':') + 1] - if is_node(node, 'suite'): # Normally a suite - node = node.children[2] # -> NEWLINE INDENT stmt - else: # ExprStmt - simple_stmt = self.parent - c = simple_stmt.parent.children - index = c.index(simple_stmt) - if not index: - return '' - node = c[index - 1] - - if is_node(node, 'simple_stmt'): - node = node.children[0] - - if node.type == 'string': - # TODO We have to check next leaves until there are no new - # leaves anymore that might be part of the docstring. A - # docstring can also look like this: ``'foo' 'bar' - # Returns a literal cleaned version of the ``Token``. - cleaned = cleandoc(literal_eval(node.value)) - # Since we want the docstr output to be always unicode, just - # force it. - if is_py3 or isinstance(cleaned, unicode): - return cleaned - else: - return unicode(cleaned, 'UTF-8', 'replace') - return '' - - -class Base(object): - """ - This is just here to have an isinstance check, which is also used on - evaluate classes. But since they have sometimes a special type of - delegation, it is important for those classes to override this method. - - I know that there is a chance to do such things with __instancecheck__, but - since Python 2.5 doesn't support it, I decided to do it this way. - """ - __slots__ = () - - def isinstance(self, *cls): - return isinstance(self, cls) - - @Python3Method - def get_parent_until(self, classes=(), reverse=False, - include_current=True): - """ - Searches the parent "chain" until the object is an instance of - classes. If classes is empty return the last parent in the chain - (is without a parent). - """ - if type(classes) not in (tuple, list): - classes = (classes,) - scope = self if include_current else self.parent - while scope.parent is not None: - # TODO why if classes? - if classes and reverse != scope.isinstance(*classes): - break - scope = scope.parent - return scope - - def get_parent_scope(self, include_flows=False): - """ - Returns the underlying scope. - """ - scope = self.parent - while scope is not None: - if include_flows and isinstance(scope, Flow): - return scope - if scope.is_scope(): - break - scope = scope.parent - return scope - - def is_scope(self): - # Default is not being a scope. Just inherit from Scope. - return False - - -class Leaf(Base): - __slots__ = ('position_modifier', 'value', 'parent', '_start_pos', 'prefix') - - def __init__(self, position_modifier, value, start_pos, prefix=''): - self.position_modifier = position_modifier - self.value = value - self._start_pos = start_pos - self.prefix = prefix - self.parent = None - - @property - def start_pos(self): - return self._start_pos[0] + self.position_modifier.line, self._start_pos[1] - - @start_pos.setter - def start_pos(self, value): - self._start_pos = value[0] - self.position_modifier.line, value[1] - - @property - def end_pos(self): - return (self._start_pos[0] + self.position_modifier.line, - self._start_pos[1] + len(self.value)) - - def move(self, line_offset, column_offset): - self._start_pos = (self._start_pos[0] + line_offset, - self._start_pos[1] + column_offset) - - def get_previous(self): - """ - Returns the previous leaf in the parser tree. - """ - node = self - while True: - c = node.parent.children - i = c.index(self) - if i == 0: - node = node.parent - if node.parent is None: - raise IndexError('Cannot access the previous element of the first one.') - else: - node = c[i - 1] - break - - while True: - try: - node = node.children[-1] - except AttributeError: # A Leaf doesn't have children. - return node - - def get_code(self): - return self.prefix + self.value - - def next_sibling(self): - """ - The node immediately following the invocant in their parent's children - list. If the invocant does not have a next sibling, it is None - """ - # Can't use index(); we need to test by identity - for i, child in enumerate(self.parent.children): - if child is self: - try: - return self.parent.children[i + 1] - except IndexError: - return None - - def prev_sibling(self): - """ - The node/leaf immediately preceding the invocant in their parent's - children list. If the invocant does not have a previous sibling, it is - None. - """ - # Can't use index(); we need to test by identity - for i, child in enumerate(self.parent.children): - if child is self: - if i == 0: - return None - return self.parent.children[i - 1] - - @utf8_repr - def __repr__(self): - return "<%s: %s>" % (type(self).__name__, self.value) - - -class LeafWithNewLines(Leaf): - __slots__ = () - - @property - def end_pos(self): - """ - Literals and whitespace end_pos are more complicated than normal - end_pos, because the containing newlines may change the indexes. - """ - end_pos_line, end_pos_col = self.start_pos - lines = self.value.split('\n') - end_pos_line += len(lines) - 1 - # Check for multiline token - if self.start_pos[0] == end_pos_line: - end_pos_col += len(lines[-1]) - else: - end_pos_col = len(lines[-1]) - return end_pos_line, end_pos_col - - -class Whitespace(LeafWithNewLines): - """Contains NEWLINE and ENDMARKER tokens.""" - __slots__ = () - type = 'whitespace' - - -class Name(Leaf): - """ - A string. Sometimes it is important to know if the string belongs to a name - or not. - """ - type = 'name' - __slots__ = () - - def __str__(self): - return self.value - - def __unicode__(self): - return self.value - - def __repr__(self): - return "<%s: %s@%s,%s>" % (type(self).__name__, self.value, - self.start_pos[0], self.start_pos[1]) - - def get_definition(self): - scope = self - while scope.parent is not None: - parent = scope.parent - if scope.isinstance(Node, Name) and parent.type != 'simple_stmt': - if scope.type == 'testlist_comp': - try: - if isinstance(scope.children[1], CompFor): - return scope.children[1] - except IndexError: - pass - scope = parent - else: - break - return scope - - def is_definition(self): - stmt = self.get_definition() - if stmt.type in ('funcdef', 'classdef', 'file_input', 'param'): - return self == stmt.name - elif stmt.type == 'for_stmt': - return self.start_pos < stmt.children[2].start_pos - elif stmt.type == 'try_stmt': - return self.prev_sibling() == 'as' - else: - return stmt.type in ('expr_stmt', 'import_name', 'import_from', - 'comp_for', 'with_stmt') \ - and self in stmt.get_defined_names() - - def assignment_indexes(self): - """ - Returns an array of ints of the indexes that are used in tuple - assignments. - - For example if the name is ``y`` in the following code:: - - x, (y, z) = 2, '' - - would result in ``[1, 0]``. - """ - indexes = [] - node = self.parent - compare = self - while node is not None: - if is_node(node, 'testlist_comp', 'testlist_star_expr', 'exprlist'): - for i, child in enumerate(node.children): - if child == compare: - indexes.insert(0, int(i / 2)) - break - else: - raise LookupError("Couldn't find the assignment.") - elif isinstance(node, (ExprStmt, CompFor)): - break - - compare = node - node = node.parent - return indexes - - -class Literal(LeafWithNewLines): - __slots__ = () - - def eval(self): - return literal_eval(self.value) - - -class Number(Literal): - type = 'number' - __slots__ = () - - -class String(Literal): - type = 'string' - __slots__ = () - - -class Operator(Leaf): - type = 'operator' - __slots__ = () - - def __str__(self): - return self.value - - def __eq__(self, other): - """ - Make comparisons with strings easy. - Improves the readability of the parser. - """ - if isinstance(other, Operator): - return self is other - else: - return self.value == other - - def __ne__(self, other): - """Python 2 compatibility.""" - return self.value != other - - def __hash__(self): - return hash(self.value) - - -class Keyword(Leaf): - type = 'keyword' - __slots__ = () - - def __eq__(self, other): - """ - Make comparisons with strings easy. - Improves the readability of the parser. - """ - if isinstance(other, Keyword): - return self is other - return self.value == other - - def __ne__(self, other): - """Python 2 compatibility.""" - return not self.__eq__(other) - - def __hash__(self): - return hash(self.value) - - -class BaseNode(Base): - """ - The super class for Scope, Import, Name and Statement. Every object in - the parser tree inherits from this class. - """ - __slots__ = ('children', 'parent') - type = None - - def __init__(self, children): - """ - Initialize :class:`BaseNode`. - - :param children: The module in which this Python object locates. - """ - for c in children: - c.parent = self - self.children = children - self.parent = None - - def move(self, line_offset, column_offset): - """ - Move the Node's start_pos. - """ - for c in self.children: - c.move(line_offset, column_offset) - - @property - def start_pos(self): - return self.children[0].start_pos - - @property - def end_pos(self): - return self.children[-1].end_pos - - def get_code(self): - return "".join(c.get_code() for c in self.children) - - @Python3Method - def name_for_position(self, position): - for c in self.children: - if isinstance(c, Leaf): - if isinstance(c, Name) and c.start_pos <= position <= c.end_pos: - return c - else: - result = c.name_for_position(position) - if result is not None: - return result - return None - - @Python3Method - def get_statement_for_position(self, pos): - for c in self.children: - if c.start_pos <= pos <= c.end_pos: - if c.type not in ('decorated', 'simple_stmt', 'suite') \ - and not isinstance(c, (Flow, ClassOrFunc)): - return c - else: - try: - return c.get_statement_for_position(pos) - except AttributeError: - pass # Must be a non-scope - return None - - def first_leaf(self): - try: - return self.children[0].first_leaf() - except AttributeError: - return self.children[0] - - @utf8_repr - def __repr__(self): - code = self.get_code().replace('\n', ' ') - if not is_py3: - code = code.encode(encoding, 'replace') - return "<%s: %s@%s,%s>" % \ - (type(self).__name__, code, self.start_pos[0], self.start_pos[1]) - - -class Node(BaseNode): - """Concrete implementation for interior nodes.""" - __slots__ = ('type',) - - def __init__(self, type, children): - """ - Initializer. - - Takes a type constant (a symbol number >= 256), a sequence of - child nodes, and an optional context keyword argument. - - As a side effect, the parent pointers of the children are updated. - """ - super(Node, self).__init__(children) - self.type = type - - def __repr__(self): - return "%s(%s, %r)" % (self.__class__.__name__, self.type, self.children) - - -class IsScopeMeta(type): - def __instancecheck__(self, other): - return other.is_scope() - - -class IsScope(use_metaclass(IsScopeMeta)): - pass - - -class Scope(BaseNode, DocstringMixin): - """ - Super class for the parser tree, which represents the state of a python - text file. - A Scope manages and owns its subscopes, which are classes and functions, as - well as variables and imports. It is used to access the structure of python - files. - - :param start_pos: The position (line and column) of the scope. - :type start_pos: tuple(int, int) - """ - __slots__ = ('names_dict',) - - def __init__(self, children): - super(Scope, self).__init__(children) - - @property - def returns(self): - # Needed here for fast_parser, because the fast_parser splits and - # returns will be in "normal" modules. - return self._search_in_scope(ReturnStmt) - - @property - def subscopes(self): - return self._search_in_scope(Scope) - - @property - def flows(self): - return self._search_in_scope(Flow) - - @property - def imports(self): - return self._search_in_scope(Import) - - @Python3Method - def _search_in_scope(self, typ): - def scan(children): - elements = [] - for element in children: - if isinstance(element, typ): - elements.append(element) - if is_node(element, 'suite', 'simple_stmt', 'decorated') \ - or isinstance(element, Flow): - elements += scan(element.children) - return elements - - return scan(self.children) - - @property - def statements(self): - return self._search_in_scope((ExprStmt, KeywordStatement)) - - def is_scope(self): - return True - - def __repr__(self): - try: - name = self.path - except AttributeError: - try: - name = self.name - except AttributeError: - name = self.command - - return "<%s: %s@%s-%s>" % (type(self).__name__, name, - self.start_pos[0], self.end_pos[0]) - - def walk(self): - yield self - for s in self.subscopes: - for scope in s.walk(): - yield scope - - for r in self.statements: - while isinstance(r, Flow): - for scope in r.walk(): - yield scope - r = r.next - - -class Module(Scope): - """ - The top scope, which is always a module. - Depending on the underlying parser this may be a full module or just a part - of a module. - """ - __slots__ = ('path', 'global_names', 'used_names', '_name', - 'error_statement_stacks') - type = 'file_input' - - def __init__(self, children): - """ - Initialize :class:`Module`. - - :type path: str - :arg path: File path to this module. - - .. todo:: Document `top_module`. - """ - super(Module, self).__init__(children) - self.path = None # Set later. - - @property - @cache.underscore_memoization - def name(self): - """ This is used for the goto functions. """ - if self.path is None: - string = '' # no path -> empty name - else: - sep = (re.escape(os.path.sep),) * 2 - r = re.search(r'([^%s]*?)(%s__init__)?(\.py|\.so)?$' % sep, self.path) - # Remove PEP 3149 names - string = re.sub('\.[a-z]+-\d{2}[mud]{0,3}$', '', r.group(1)) - # Positions are not real, but a module starts at (1, 0) - p = (1, 0) - name = Name(zero_position_modifier, string, p) - name.parent = self - return name - - @property - def has_explicit_absolute_import(self): - """ - Checks if imports in this module are explicitly absolute, i.e. there - is a ``__future__`` import. - """ - # TODO this is a strange scan and not fully correct. I think Python's - # parser does it in a different way and scans for the first - # statement/import with a tokenizer (to check for syntax changes like - # the future print statement). - for imp in self.imports: - if imp.type == 'import_from' and imp.level == 0: - for path in imp.paths(): - if [str(name) for name in path] == ['__future__', 'absolute_import']: - return True - return False - - -class Decorator(BaseNode): - type = 'decorator' - __slots__ = () - - -class ClassOrFunc(Scope): - __slots__ = () - - @property - def name(self): - return self.children[1] - - def get_decorators(self): - decorated = self.parent - if is_node(decorated, 'decorated'): - if is_node(decorated.children[0], 'decorators'): - return decorated.children[0].children - else: - return decorated.children[:1] - else: - return [] - - -class Class(ClassOrFunc): - """ - Used to store the parsed contents of a python class. - - :param name: The Class name. - :type name: str - :param supers: The super classes of a Class. - :type supers: list - :param start_pos: The start position (line, column) of the class. - :type start_pos: tuple(int, int) - """ - type = 'classdef' - __slots__ = () - - def __init__(self, children): - super(Class, self).__init__(children) - - def get_super_arglist(self): - if self.children[2] != '(': # Has no parentheses - return None - else: - if self.children[3] == ')': # Empty parentheses - return None - else: - return self.children[3] - - @property - def doc(self): - """ - Return a document string including call signature of __init__. - """ - docstr = self.raw_doc - for sub in self.subscopes: - if str(sub.name) == '__init__': - return '%s\n\n%s' % ( - sub.get_call_signature(func_name=self.name), docstr) - return docstr - - -def _create_params(parent, argslist_list): - """ - `argslist_list` is a list that can contain an argslist as a first item, but - most not. It's basically the items between the parameter brackets (which is - at most one item). - This function modifies the parser structure. It generates `Param` objects - from the normal ast. Those param objects do not exist in a normal ast, but - make the evaluation of the ast tree so much easier. - You could also say that this function replaces the argslist node with a - list of Param objects. - """ - def check_python2_nested_param(node): - """ - Python 2 allows params to look like ``def x(a, (b, c))``, which is - basically a way of unpacking tuples in params. Python 3 has ditched - this behavior. Jedi currently just ignores those constructs. - """ - return node.type == 'tfpdef' and node.children[0] == '(' - - try: - first = argslist_list[0] - except IndexError: - return [] - - if first.type in ('name', 'tfpdef'): - if check_python2_nested_param(first): - return [] - else: - return [Param([first], parent)] - else: # argslist is a `typedargslist` or a `varargslist`. - children = first.children - params = [] - start = 0 - # Start with offset 1, because the end is higher. - for end, child in enumerate(children + [None], 1): - if child is None or child == ',': - new_children = children[start:end] - if new_children: # Could as well be comma and then end. - if check_python2_nested_param(new_children[0]): - continue - params.append(Param(new_children, parent)) - start = end - return params - - -class Function(ClassOrFunc): - """ - Used to store the parsed contents of a python function. - """ - __slots__ = ('listeners',) - type = 'funcdef' - - def __init__(self, children): - super(Function, self).__init__(children) - self.listeners = set() # not used here, but in evaluation. - parameters = self.children[2] # After `def foo` - parameters.children[1:-1] = _create_params(parameters, parameters.children[1:-1]) - - @property - def params(self): - return self.children[2].children[1:-1] - - @property - def name(self): - return self.children[1] # First token after `def` - - @property - def yields(self): - # TODO This is incorrect, yields are also possible in a statement. - return self._search_in_scope(YieldExpr) - - def is_generator(self): - return bool(self.yields) - - def annotation(self): - try: - return self.children[6] # 6th element: def foo(...) -> bar - except IndexError: - return None - - def get_call_signature(self, width=72, func_name=None): - """ - Generate call signature of this function. - - :param width: Fold lines if a line is longer than this value. - :type width: int - :arg func_name: Override function name when given. - :type func_name: str - - :rtype: str - """ - func_name = func_name or self.children[1] - code = unicode(func_name) + self.children[2].get_code() - return '\n'.join(textwrap.wrap(code, width)) - - @property - def doc(self): - """ Return a document string including call signature. """ - docstr = self.raw_doc - return '%s\n\n%s' % (self.get_call_signature(), docstr) - - -class Lambda(Function): - """ - Lambdas are basically trimmed functions, so give it the same interface. - """ - type = 'lambda' - __slots__ = () - - def __init__(self, children): - # We don't want to call the Function constructor, call its parent. - super(Function, self).__init__(children) - self.listeners = set() # not used here, but in evaluation. - lst = self.children[1:-2] # After `def foo` - self.children[1:-2] = _create_params(self, lst) - - @property - def params(self): - return self.children[1:-2] - - def is_generator(self): - return False - - def yields(self): - return [] - - def __repr__(self): - return "<%s@%s>" % (self.__class__.__name__, self.start_pos) - - -class Flow(BaseNode): - __slots__ = () - - -class IfStmt(Flow): - type = 'if_stmt' - __slots__ = () - - def check_nodes(self): - """ - Returns all the `test` nodes that are defined as x, here: - - if x: - pass - elif x: - pass - """ - for i, c in enumerate(self.children): - if c in ('elif', 'if'): - yield self.children[i + 1] - - def node_in_which_check_node(self, node): - for check_node in reversed(list(self.check_nodes())): - if check_node.start_pos < node.start_pos: - return check_node - - def node_after_else(self, node): - """ - Checks if a node is defined after `else`. - """ - for c in self.children: - if c == 'else': - if node.start_pos > c.start_pos: - return True - else: - return False - - -class WhileStmt(Flow): - type = 'while_stmt' - __slots__ = () - - -class ForStmt(Flow): - type = 'for_stmt' - __slots__ = () - - -class TryStmt(Flow): - type = 'try_stmt' - __slots__ = () - - def except_clauses(self): - """ - Returns the ``test`` nodes found in ``except_clause`` nodes. - Returns ``[None]`` for except clauses without an exception given. - """ - for node in self.children: - if node.type == 'except_clause': - yield node.children[1] - elif node == 'except': - yield None - - -class WithStmt(Flow): - type = 'with_stmt' - __slots__ = () - - def get_defined_names(self): - names = [] - for with_item in self.children[1:-2:2]: - # Check with items for 'as' names. - if is_node(with_item, 'with_item'): - names += _defined_names(with_item.children[2]) - return names - - def node_from_name(self, name): - node = name - while True: - node = node.parent - if is_node(node, 'with_item'): - return node.children[0] - - -class Import(BaseNode): - __slots__ = () - - def path_for_name(self, name): - try: - # The name may be an alias. If it is, just map it back to the name. - name = self.aliases()[name] - except KeyError: - pass - - for path in self.paths(): - if name in path: - return path[:path.index(name) + 1] - raise ValueError('Name should be defined in the import itself') - - def is_nested(self): - return False # By default, sub classes may overwrite this behavior - - def is_star_import(self): - return self.children[-1] == '*' - - -class ImportFrom(Import): - type = 'import_from' - __slots__ = () - - def get_defined_names(self): - return [alias or name for name, alias in self._as_name_tuples()] - - def aliases(self): - """Mapping from alias to its corresponding name.""" - return dict((alias, name) for name, alias in self._as_name_tuples() - if alias is not None) - - def get_from_names(self): - for n in self.children[1:]: - if n not in ('.', '...'): - break - if is_node(n, 'dotted_name'): # from x.y import - return n.children[::2] - elif n == 'import': # from . import - return [] - else: # from x import - return [n] - - @property - def level(self): - """The level parameter of ``__import__``.""" - level = 0 - for n in self.children[1:]: - if n in ('.', '...'): - level += len(n.value) - else: - break - return level - - def _as_name_tuples(self): - last = self.children[-1] - if last == ')': - last = self.children[-2] - elif last == '*': - return # No names defined directly. - - if is_node(last, 'import_as_names'): - as_names = last.children[::2] - else: - as_names = [last] - for as_name in as_names: - if as_name.type == 'name': - yield as_name, None - else: - yield as_name.children[::2] # yields x, y -> ``x as y`` - - def star_import_name(self): - """ - The last name defined in a star import. - """ - return self.paths()[-1][-1] - - def paths(self): - """ - The import paths defined in an import statement. Typically an array - like this: ``[, ]``. - """ - dotted = self.get_from_names() - - if self.children[-1] == '*': - return [dotted] - return [dotted + [name] for name, alias in self._as_name_tuples()] - - -class ImportName(Import): - """For ``import_name`` nodes. Covers normal imports without ``from``.""" - type = 'import_name' - __slots__ = () - - def get_defined_names(self): - return [alias or path[0] for path, alias in self._dotted_as_names()] - - @property - def level(self): - """The level parameter of ``__import__``.""" - return 0 # Obviously 0 for imports without from. - - def paths(self): - return [path for path, alias in self._dotted_as_names()] - - def _dotted_as_names(self): - """Generator of (list(path), alias) where alias may be None.""" - dotted_as_names = self.children[1] - if is_node(dotted_as_names, 'dotted_as_names'): - as_names = dotted_as_names.children[::2] - else: - as_names = [dotted_as_names] - - for as_name in as_names: - if is_node(as_name, 'dotted_as_name'): - alias = as_name.children[2] - as_name = as_name.children[0] - else: - alias = None - if as_name.type == 'name': - yield [as_name], alias - else: - # dotted_names - yield as_name.children[::2], alias - - def is_nested(self): - """ - This checks for the special case of nested imports, without aliases and - from statement:: - - import foo.bar - """ - return [1 for path, alias in self._dotted_as_names() - if alias is None and len(path) > 1] - - def aliases(self): - return dict((alias, path[-1]) for path, alias in self._dotted_as_names() - if alias is not None) - - -class KeywordStatement(BaseNode): - """ - For the following statements: `assert`, `del`, `global`, `nonlocal`, - `raise`, `return`, `yield`, `pass`, `continue`, `break`, `return`, `yield`. - """ - __slots__ = () - - @property - def keyword(self): - return self.children[0].value - - -class AssertStmt(KeywordStatement): - type = 'assert_stmt' - __slots__ = () - - def assertion(self): - return self.children[1] - - -class GlobalStmt(KeywordStatement): - type = 'global_stmt' - __slots__ = () - - def get_defined_names(self): - return [] - - def get_global_names(self): - return self.children[1::2] - - -class ReturnStmt(KeywordStatement): - type = 'return_stmt' - __slots__ = () - - -class YieldExpr(BaseNode): - type = 'yield_expr' - __slots__ = () - - -def _defined_names(current): - """ - A helper function to find the defined names in statements, for loops and - list comprehensions. - """ - names = [] - if is_node(current, 'testlist_star_expr', 'testlist_comp', 'exprlist'): - for child in current.children[::2]: - names += _defined_names(child) - elif is_node(current, 'atom'): - names += _defined_names(current.children[1]) - elif is_node(current, 'power'): - if current.children[-2] != '**': # Just if there's no operation - trailer = current.children[-1] - if trailer.children[0] == '.': - names.append(trailer.children[1]) - else: - names.append(current) - return names - - -class ExprStmt(BaseNode, DocstringMixin): - type = 'expr_stmt' - __slots__ = () - - def get_defined_names(self): - return list(chain.from_iterable(_defined_names(self.children[i]) - for i in range(0, len(self.children) - 2, 2) - if '=' in self.children[i + 1].value)) - - def get_rhs(self): - """Returns the right-hand-side of the equals.""" - return self.children[-1] - - def first_operation(self): - """ - Returns `+=`, `=`, etc or None if there is no operation. - """ - try: - return self.children[1] - except IndexError: - return None - - -class Param(BaseNode): - """ - It's a helper class that makes business logic with params much easier. The - Python grammar defines no ``param`` node. It defines it in a different way - that is not really suited to working with parameters. - """ - type = 'param' - - def __init__(self, children, parent): - super(Param, self).__init__(children) - self.parent = parent - for child in children: - child.parent = self - - @property - def stars(self): - first = self.children[0] - if first in ('*', '**'): - return len(first.value) - return 0 - - @property - def default(self): - try: - return self.children[int(self.children[0] in ('*', '**')) + 2] - except IndexError: - return None - - def annotation(self): - # Generate from tfpdef. - raise NotImplementedError - - def _tfpdef(self): - """ - tfpdef: see grammar.txt. - """ - offset = int(self.children[0] in ('*', '**')) - return self.children[offset] - - @property - def name(self): - if is_node(self._tfpdef(), 'tfpdef'): - return self._tfpdef().children[0] - else: - return self._tfpdef() - - @property - def position_nr(self): - return self.parent.children.index(self) - 1 - - @property - def parent_function(self): - return self.get_parent_until(IsScope) - - def __repr__(self): - default = '' if self.default is None else '=%s' % self.default - return '<%s: %s>' % (type(self).__name__, str(self._tfpdef()) + default) - - -class CompFor(BaseNode): - type = 'comp_for' - __slots__ = () - - def is_scope(self): - return True - - @property - def names_dict(self): - dct = {} - for name in self.get_defined_names(): - arr = dct.setdefault(name.value, []) - arr.append(name) - return dct - - def names_dicts(self, search_global): - yield self.names_dict - - def get_defined_names(self): - return _defined_names(self.children[1]) diff --git a/pythonFiles/release/jedi/parser/user_context.py b/pythonFiles/release/jedi/parser/user_context.py deleted file mode 100755 index 3cb24a7ceeb5..000000000000 --- a/pythonFiles/release/jedi/parser/user_context.py +++ /dev/null @@ -1,339 +0,0 @@ -import re -import os -import keyword - -from jedi import cache -from jedi import common -from jedi.parser import tokenize, Parser -from jedi._compatibility import u -from jedi.parser.fast import FastParser -from jedi.parser import tree -from jedi import debug -from jedi.common import PushBackIterator - - -REPLACE_STR = r"[bBuU]?[rR]?" + (r"(?:(')[^\n'\\]*(?:\\.[^\n'\\]*)*(?:'|$)" + - '|' + - r'(")[^\n"\\]*(?:\\.[^\n"\\]*)*(?:"|$))') -REPLACE_STR = re.compile(REPLACE_STR) - - -class UserContext(object): - """ - :param source: The source code of the file. - :param position: The position, the user is currently in. Only important \ - for the main file. - """ - def __init__(self, source, position): - self.source = source - self.position = position - self._line_cache = None - - self._relevant_temp = None - - @cache.underscore_memoization - def get_path_until_cursor(self): - """ Get the path under the cursor. """ - path, self._start_cursor_pos = self._calc_path_until_cursor(self.position) - return path - - def _backwards_line_generator(self, start_pos): - self._line_temp, self._column_temp = start_pos - first_line = self.get_line(start_pos[0])[:self._column_temp] - - self._line_length = self._column_temp - yield first_line[::-1] + '\n' - - while True: - self._line_temp -= 1 - line = self.get_line(self._line_temp) - self._line_length = len(line) - yield line[::-1] + '\n' - - def _get_backwards_tokenizer(self, start_pos, line_gen=None): - if line_gen is None: - line_gen = self._backwards_line_generator(start_pos) - token_gen = tokenize.generate_tokens(lambda: next(line_gen)) - for typ, tok_str, tok_start_pos, prefix in token_gen: - line = self.get_line(self._line_temp) - # Calculate the real start_pos of the token. - if tok_start_pos[0] == 1: - # We are in the first checked line - column = start_pos[1] - tok_start_pos[1] - else: - column = len(line) - tok_start_pos[1] - # Multi-line docstrings must be accounted for. - first_line = common.splitlines(tok_str)[0] - column -= len(first_line) - # Reverse the token again, so that it is in normal order again. - yield typ, tok_str[::-1], (self._line_temp, column), prefix[::-1] - - def _calc_path_until_cursor(self, start_pos): - """ - Something like a reverse tokenizer that tokenizes the reversed strings. - """ - open_brackets = ['(', '[', '{'] - close_brackets = [')', ']', '}'] - - start_cursor = start_pos - gen = PushBackIterator(self._get_backwards_tokenizer(start_pos)) - string = u('') - level = 0 - force_point = False - last_type = None - is_first = True - for tok_type, tok_str, tok_start_pos, prefix in gen: - if is_first: - if prefix: # whitespace is not a path - return u(''), start_cursor - is_first = False - - if last_type == tok_type == tokenize.NAME: - string = ' ' + string - - if level: - if tok_str in close_brackets: - level += 1 - elif tok_str in open_brackets: - level -= 1 - elif tok_str == '.': - force_point = False - elif force_point: - # Reversed tokenizing, therefore a number is recognized as a - # floating point number. - # The same is true for string prefixes -> represented as a - # combination of string and name. - if tok_type == tokenize.NUMBER and tok_str[-1] == '.' \ - or tok_type == tokenize.NAME and last_type == tokenize.STRING \ - and tok_str.lower() in ('b', 'u', 'r', 'br', 'ur'): - force_point = False - else: - break - elif tok_str in close_brackets: - level += 1 - elif tok_type in [tokenize.NAME, tokenize.STRING]: - if keyword.iskeyword(tok_str) and string: - # If there's already something in the string, a keyword - # never adds any meaning to the current statement. - break - force_point = True - elif tok_type == tokenize.NUMBER: - pass - else: - if tok_str == '-': - next_tok = next(gen) - if next_tok[1] == 'e': - gen.push_back(next_tok) - else: - break - else: - break - - start_cursor = tok_start_pos - string = tok_str + prefix + string - last_type = tok_type - - # Don't need whitespace around a statement. - return string.strip(), start_cursor - - def get_path_under_cursor(self): - """ - Return the path under the cursor. If there is a rest of the path left, - it will be added to the stuff before it. - """ - return self.get_path_until_cursor() + self.get_path_after_cursor() - - def get_path_after_cursor(self): - line = self.get_line(self.position[0]) - return re.search("[\w\d]*", line[self.position[1]:]).group(0) - - def get_operator_under_cursor(self): - line = self.get_line(self.position[0]) - after = re.match("[^\w\s]+", line[self.position[1]:]) - before = re.match("[^\w\s]+", line[:self.position[1]][::-1]) - return (before.group(0) if before is not None else '') \ - + (after.group(0) if after is not None else '') - - def call_signature(self): - """ - :return: Tuple of string of the call and the index of the cursor. - """ - def get_line(pos): - def simplify_str(match): - """ - To avoid having strings without end marks (error tokens) and - strings that just screw up all the call signatures, just - simplify everything. - """ - mark = match.group(1) or match.group(2) - return mark + ' ' * (len(match.group(0)) - 2) + mark - - line_gen = self._backwards_line_generator(pos) - for line in line_gen: - # We have to switch the already backwards lines twice, because - # we scan them from start. - line = line[::-1] - modified = re.sub(REPLACE_STR, simplify_str, line) - yield modified[::-1] - - index = 0 - level = 0 - next_must_be_name = False - next_is_key = False - key_name = None - generator = self._get_backwards_tokenizer(self.position, get_line(self.position)) - for tok_type, tok_str, start_pos, prefix in generator: - if tok_str in tokenize.ALWAYS_BREAK_TOKENS: - break - elif next_must_be_name: - if tok_type == tokenize.NUMBER: - # If there's a number at the end of the string, it will be - # tokenized as a number. So add it to the name. - tok_type, t, _, _ = next(generator) - if tok_type == tokenize.NAME: - end_pos = start_pos[0], start_pos[1] + len(tok_str) - call, start_pos = self._calc_path_until_cursor(start_pos=end_pos) - return call, index, key_name, start_pos - index = 0 - next_must_be_name = False - elif next_is_key: - if tok_type == tokenize.NAME: - key_name = tok_str - next_is_key = False - - if tok_str == '(': - level += 1 - if level == 1: - next_must_be_name = True - level = 0 - elif tok_str == ')': - level -= 1 - elif tok_str == ',': - index += 1 - elif tok_str == '=': - next_is_key = True - return None, 0, None, (0, 0) - - def get_context(self, yield_positions=False): - self.get_path_until_cursor() # In case _start_cursor_pos is undefined. - pos = self._start_cursor_pos - while True: - # remove non important white space - line = self.get_line(pos[0]) - while True: - if pos[1] == 0: - line = self.get_line(pos[0] - 1) - if line and line[-1] == '\\': - pos = pos[0] - 1, len(line) - 1 - continue - else: - break - - if line[pos[1] - 1].isspace(): - pos = pos[0], pos[1] - 1 - else: - break - - try: - result, pos = self._calc_path_until_cursor(start_pos=pos) - if yield_positions: - yield pos - else: - yield result - except StopIteration: - if yield_positions: - yield None - else: - yield '' - - def get_line(self, line_nr): - if not self._line_cache: - self._line_cache = common.splitlines(self.source) - - if line_nr == 0: - # This is a fix for the zeroth line. We need a newline there, for - # the backwards parser. - return u('') - if line_nr < 0: - raise StopIteration() - try: - return self._line_cache[line_nr - 1] - except IndexError: - raise StopIteration() - - def get_position_line(self): - return self.get_line(self.position[0])[:self.position[1]] - - -class UserContextParser(object): - def __init__(self, grammar, source, path, position, user_context, - parser_done_callback, use_fast_parser=True): - self._grammar = grammar - self._source = source - self._path = path and os.path.abspath(path) - self._position = position - self._user_context = user_context - self._use_fast_parser = use_fast_parser - self._parser_done_callback = parser_done_callback - - @cache.underscore_memoization - def _parser(self): - cache.invalidate_star_import_cache(self._path) - if self._use_fast_parser: - parser = FastParser(self._grammar, self._source, self._path) - # Don't pickle that module, because the main module is changing quickly - cache.save_parser(self._path, parser, pickling=False) - else: - parser = Parser(self._grammar, self._source, self._path) - self._parser_done_callback(parser) - return parser - - @cache.underscore_memoization - def user_stmt(self): - module = self.module() - debug.speed('parsed') - return module.get_statement_for_position(self._position) - - @cache.underscore_memoization - def user_stmt_with_whitespace(self): - """ - Returns the statement under the cursor even if the statement lies - before the cursor. - """ - user_stmt = self.user_stmt() - - if not user_stmt: - # for statements like `from x import ` (cursor not in statement) - # or `abs( ` where the cursor is out in the whitespace. - if self._user_context.get_path_under_cursor(): - # We really should have a user_stmt, but the parser couldn't - # process it - probably a Syntax Error (or in a comment). - debug.warning('No statement under the cursor.') - return - pos = next(self._user_context.get_context(yield_positions=True)) - user_stmt = self.module().get_statement_for_position(pos) - return user_stmt - - @cache.underscore_memoization - def user_scope(self): - """ - Returns the scope in which the user resides. This includes flows. - """ - user_stmt = self.user_stmt() - if user_stmt is None: - def scan(scope): - for s in scope.children: - if s.start_pos <= self._position <= s.end_pos: - if isinstance(s, (tree.Scope, tree.Flow)): - if isinstance(s, tree.Flow): - return s - return scan(s) or s - elif s.type in ('suite', 'decorated'): - return scan(s) - - return scan(self.module()) or self.module() - else: - return user_stmt.get_parent_scope(include_flows=True) - - def module(self): - return self._parser().module diff --git a/src/client/language/tokenizer.ts b/src/client/language/tokenizer.ts index 0a2160fc15c5..bf20fb3f44ba 100644 --- a/src/client/language/tokenizer.ts +++ b/src/client/language/tokenizer.ts @@ -59,7 +59,7 @@ export class Tokenizer implements ITokenizer { throw new Error('Invalid range length'); } - this.mode = mode ? mode : TokenizerMode.Full; + this.mode = mode !== undefined ? mode : TokenizerMode.Full; this.cs = new CharacterStream(text); this.cs.position = start; @@ -127,6 +127,10 @@ export class Tokenizer implements ITokenizer { case Char.Colon: this.tokens.push(new Token(TokenType.Colon, this.cs.position, 1)); break; + case Char.Period: + case Char.At: + this.tokens.push(new Token(TokenType.Unknown, this.cs.position, 1)); + break; default: if (this.isPossibleNumber()) { if (this.tryNumber()) { diff --git a/src/client/providers/completionProvider.ts b/src/client/providers/completionProvider.ts index ba18829db73d..fb0ae33bfb1a 100644 --- a/src/client/providers/completionProvider.ts +++ b/src/client/providers/completionProvider.ts @@ -33,8 +33,7 @@ export class PythonCompletionItemProvider implements vscode.CompletionItemProvid if (!item.documentation) { const itemInfos = await this.completionSource.getDocumentation(item, token); if (itemInfos && itemInfos.length > 0) { - item.detail = itemInfos[0].detail; - item.documentation = itemInfos[0].documentation; + item.documentation = itemInfos[0].tooltip; } } return item; diff --git a/src/client/providers/itemInfoSource.ts b/src/client/providers/itemInfoSource.ts index b78515c1822f..b851f61b533d 100644 --- a/src/client/providers/itemInfoSource.ts +++ b/src/client/providers/itemInfoSource.ts @@ -7,13 +7,12 @@ import * as vscode from 'vscode'; import { RestTextConverter } from '../common/markdown/restTextConverter'; import { JediFactory } from '../languageServices/jediProxyFactory'; import * as proxy from './jediProxy'; -import { IHoverItem } from './jediProxy'; export class LanguageItemInfo { constructor( public tooltip: vscode.MarkdownString, public detail: string, - public documentation: vscode.MarkdownString) { } + public signature: vscode.MarkdownString) { } } export class ItemInfoSource { @@ -83,14 +82,12 @@ export class ItemInfoSource { private getItemInfoFromHoverResult(data: proxy.IHoverResult, currentWord: string): LanguageItemInfo[] { const infos: LanguageItemInfo[] = []; - const capturedInfo: string[] = []; data.items.forEach(item => { const signature = this.getSignature(item, currentWord); let tooltip = new vscode.MarkdownString(); if (item.docstring) { let lines = item.docstring.split(/\r?\n/); - const dnd = this.getDetailAndDescription(item, lines); // If the docstring starts with the signature, then remove those lines from the docstring. if (lines.length > 0 && item.signature.indexOf(lines[0]) === 0) { @@ -100,11 +97,10 @@ export class ItemInfoSource { lines = lines.filter((line, index) => index > endIndex); } } - if (lines.length > 0 && item.signature.startsWith(currentWord) && lines[0].startsWith(currentWord) && lines[0].endsWith(')')) { + if (lines.length > 0 && currentWord.length > 0 && item.signature.startsWith(currentWord) && lines[0].startsWith(currentWord) && lines[0].endsWith(')')) { lines.shift(); } - // Tooltip is only used in hover if (signature.length > 0) { tooltip = tooltip.appendMarkdown(['```python', signature, '```', ''].join(EOL)); } @@ -112,16 +108,7 @@ export class ItemInfoSource { const description = this.textConverter.toMarkdown(lines.join(EOL)); tooltip = tooltip.appendMarkdown(description); - const documentation = this.textConverter.toMarkdown(dnd[1]); // Used only in completion list - infos.push(new LanguageItemInfo(tooltip, dnd[0], new vscode.MarkdownString(documentation))); - - const key = signature + lines.join(''); - // Sometimes we have duplicate documentation, one with a period at the end. - if (capturedInfo.indexOf(key) >= 0 || capturedInfo.indexOf(`${key}.`) >= 0) { - return; - } - capturedInfo.push(key); - capturedInfo.push(`${key}.`); + infos.push(new LanguageItemInfo(tooltip, item.description, new vscode.MarkdownString(signature))); return; } @@ -131,40 +118,21 @@ export class ItemInfoSource { } const description = this.textConverter.toMarkdown(item.description); tooltip.appendMarkdown(description); - - const lines = item.description.split(EOL); - const dd = this.getDetailAndDescription(item, lines); - const documentation = this.textConverter.escapeMarkdown(dd[1]); - infos.push(new LanguageItemInfo(tooltip, dd[0], new vscode.MarkdownString(documentation))); - - const key = signature + lines.join(''); - // Sometimes we have duplicate documentation, one with a period at the end. - if (capturedInfo.indexOf(key) >= 0 || capturedInfo.indexOf(`${key}.`) >= 0) { - return; - } - - capturedInfo.push(key); - capturedInfo.push(`${key}.`); + infos.push(new LanguageItemInfo(tooltip, item.description, new vscode.MarkdownString(signature))); return; } + + if (item.text) { // Most probably variable type + const code = currentWord && currentWord.length > 0 + ? `${currentWord}: ${item.text}` + : item.text; + tooltip.appendMarkdown(['```python', code, '```', ''].join(EOL)); + infos.push(new LanguageItemInfo(tooltip, '', new vscode.MarkdownString())); + } }); return infos; } - private getDetailAndDescription(item: IHoverItem, lines: string[]): [string, string] { - let detail: string; - let description: string; - - if (item.signature && item.signature.length > 0 && lines.length > 0 && lines[0].indexOf(item.signature) >= 0) { - detail = lines.length > 0 ? lines[0] : ''; - description = lines.filter((line, index) => index > 0).join(EOL).trim(); - } else { - detail = item.description; - description = lines.join(EOL).trim(); - } - return [detail, description]; - } - private getSignature(item: proxy.IHoverItem, currentWord: string): string { let { signature } = item; switch (item.kind) { diff --git a/src/client/providers/jediProxy.ts b/src/client/providers/jediProxy.ts index df2c537a53c0..194134b949d8 100644 --- a/src/client/providers/jediProxy.ts +++ b/src/client/providers/jediProxy.ts @@ -288,16 +288,7 @@ export class JediProxy implements vscode.Disposable { this.languageServerStarted = createDeferred(); const pythonProcess = await this.serviceContainer.get(IPythonExecutionFactory).create(Uri.file(this.workspacePath)); const args = ['completion.py']; - if (typeof this.pythonSettings.jediPath !== 'string' || this.pythonSettings.jediPath.length === 0) { - if (Array.isArray(this.pythonSettings.devOptions) && - this.pythonSettings.devOptions.some(item => item.toUpperCase().trim() === 'USERELEASEAUTOCOMP')) { - // Use standard version of jedi. - args.push('std'); - } else { - // Use preview version of jedi. - args.push('preview'); - } - } else { + if (typeof this.pythonSettings.jediPath === 'string' && this.pythonSettings.jediPath.length > 0) { args.push('custom'); args.push(this.pythonSettings.jediPath); } diff --git a/src/test/autocomplete/base.test.ts b/src/test/autocomplete/base.test.ts index 100e009e042f..bf07ac4fd783 100644 --- a/src/test/autocomplete/base.test.ts +++ b/src/test/autocomplete/base.test.ts @@ -217,6 +217,5 @@ function checkDocumentation(item: vscode.CompletionItem, expectedContains: strin assert.notEqual(documentation, null, 'Documentation is not MarkdownString'); const inDoc = documentation.value.indexOf(expectedContains) >= 0; - const inDetails = item.detail!.indexOf(expectedContains) >= 0; - assert.equal(inDoc !== inDetails, true, 'Documentation incorrect'); + assert.equal(inDoc, true, 'Documentation incorrect'); } diff --git a/src/test/definitions/hover.test.ts b/src/test/definitions/hover.test.ts index 0abda7ea51c8..ba194a902446 100644 --- a/src/test/definitions/hover.test.ts +++ b/src/test/definitions/hover.test.ts @@ -228,9 +228,6 @@ suite('Hover Definition', () => { assert.equal(normalizeMarkedString(def[0].contents[0]), '```python' + EOL + 'class Thread(group=None, target=None, name=None, args=(), kwargs=None, verbose=None)' + EOL + '```' + EOL + - 'Thread(self, group=None, target=None, name=None,' + EOL + - 'args=(), kwargs=None, verbose=None)' + EOL + - '' + EOL + 'A class that represents a thread of control.' + EOL + '' + EOL + 'This class can be safely subclassed in a limited fashion.', 'Invalid content items'); @@ -254,11 +251,8 @@ suite('Hover Definition', () => { if (contents.indexOf('```python') === -1) { assert.fail(contents, '', 'First line is incorrect', 'compare'); } - if (contents.indexOf('Random number generator base class used by bound module functions.') === -1) { - assert.fail(contents, '', '\'Random number generator\' message missing', 'compare'); - } - if (contents.indexOf('Class Random can also be subclassed if you want to use a different basic') === -1) { - assert.fail(contents, '', '\'Class Random message\' missing', 'compare'); + if (contents.indexOf('rnd: Random') === -1) { + assert.fail(contents, '', 'Variable name or type are missing', 'compare'); } }).then(done, done); }); diff --git a/src/test/format/extension.format.test.ts b/src/test/format/extension.format.test.ts index 03a44962bb06..8a9d09dff517 100644 --- a/src/test/format/extension.format.test.ts +++ b/src/test/format/extension.format.test.ts @@ -1,4 +1,3 @@ -import * as assert from 'assert'; import * as fs from 'fs-extra'; import * as path from 'path'; import * as vscode from 'vscode'; @@ -8,6 +7,7 @@ import { AutoPep8Formatter } from '../../client/formatters/autoPep8Formatter'; import { YapfFormatter } from '../../client/formatters/yapfFormatter'; import { closeActiveWindows, initialize, initializeTest } from '../initialize'; import { MockProcessService } from '../mocks/proc'; +import { compareFiles } from '../textUtils'; import { UnitTestIocContainer } from '../unittests/serviceRegistry'; const ch = vscode.window.createOutputChannel('Tests'); @@ -92,7 +92,7 @@ suite('Formatting', () => { await textEditor.edit(editBuilder => { edits.forEach(edit => editBuilder.replace(edit.range, edit.newText)); }); - assert.equal(textEditor.document.getText(), formattedContents, 'Formatted text is not the same'); + compareFiles(formattedContents, textEditor.document.getText()); } test('AutoPep8', async () => await testFormatting(new AutoPep8Formatter(ioc.serviceContainer), formattedAutoPep8, autoPep8FileToFormat, 'autopep8.output')); diff --git a/src/test/language/tokenizer.test.ts b/src/test/language/tokenizer.test.ts index 86deb9282249..c397ffec95e5 100644 --- a/src/test/language/tokenizer.test.ts +++ b/src/test/language/tokenizer.test.ts @@ -76,11 +76,105 @@ suite('Language.Tokenizer', () => { assert.equal(tokens.getItemAt(i).type, TokenType.Comment); } }); + test('Period/At to unknown token', async () => { + const t = new Tokenizer(); + const tokens = t.tokenize('.@x'); + assert.equal(tokens.count, 3); + + assert.equal(tokens.getItemAt(0).type, TokenType.Unknown); + assert.equal(tokens.getItemAt(1).type, TokenType.Unknown); + assert.equal(tokens.getItemAt(2).type, TokenType.Identifier); + }); test('Unknown token', async () => { const t = new Tokenizer(); - const tokens = t.tokenize('.'); + const tokens = t.tokenize('~$'); assert.equal(tokens.count, 1); assert.equal(tokens.getItemAt(0).type, TokenType.Unknown); }); + test('Hex number', async () => { + const t = new Tokenizer(); + const tokens = t.tokenize('1 0X2 0x3 0x'); + assert.equal(tokens.count, 4); + + assert.equal(tokens.getItemAt(0).type, TokenType.Number); + assert.equal(tokens.getItemAt(0).length, 1); + + assert.equal(tokens.getItemAt(1).type, TokenType.Number); + assert.equal(tokens.getItemAt(1).length, 3); + + assert.equal(tokens.getItemAt(2).type, TokenType.Number); + assert.equal(tokens.getItemAt(2).length, 3); + + assert.equal(tokens.getItemAt(3).type, TokenType.Unknown); + assert.equal(tokens.getItemAt(3).length, 2); + }); + test('Binary number', async () => { + const t = new Tokenizer(); + const tokens = t.tokenize('1 0B1 0b010 0b3 0b'); + assert.equal(tokens.count, 6); + + assert.equal(tokens.getItemAt(0).type, TokenType.Number); + assert.equal(tokens.getItemAt(0).length, 1); + + assert.equal(tokens.getItemAt(1).type, TokenType.Number); + assert.equal(tokens.getItemAt(1).length, 3); + + assert.equal(tokens.getItemAt(2).type, TokenType.Number); + assert.equal(tokens.getItemAt(2).length, 5); + + assert.equal(tokens.getItemAt(3).type, TokenType.Number); + assert.equal(tokens.getItemAt(3).length, 1); + + assert.equal(tokens.getItemAt(4).type, TokenType.Identifier); + assert.equal(tokens.getItemAt(4).length, 2); + + assert.equal(tokens.getItemAt(5).type, TokenType.Unknown); + assert.equal(tokens.getItemAt(5).length, 2); + }); + test('Octal number', async () => { + const t = new Tokenizer(); + const tokens = t.tokenize('1 0o4 0o077 0o9 0oO'); + assert.equal(tokens.count, 6); + + assert.equal(tokens.getItemAt(0).type, TokenType.Number); + assert.equal(tokens.getItemAt(0).length, 1); + + assert.equal(tokens.getItemAt(1).type, TokenType.Number); + assert.equal(tokens.getItemAt(1).length, 3); + + assert.equal(tokens.getItemAt(2).type, TokenType.Number); + assert.equal(tokens.getItemAt(2).length, 5); + + assert.equal(tokens.getItemAt(3).type, TokenType.Number); + assert.equal(tokens.getItemAt(3).length, 1); + + assert.equal(tokens.getItemAt(4).type, TokenType.Identifier); + assert.equal(tokens.getItemAt(4).length, 2); + + assert.equal(tokens.getItemAt(5).type, TokenType.Unknown); + assert.equal(tokens.getItemAt(5).length, 3); + }); + test('Operators', async () => { + const text = '< <> << <<= ' + + '== != > >> >>= ' + + '+ -' + + '* ** / /= //=' + + '*= += -= **= ' + + '& &= | |= ^ ^='; + const tokens = new Tokenizer().tokenize(text); + const lengths = [ + 1, 2, 2, 3, + 2, 2, 1, 2, 3, + 1, 1, + 1, 2, 1, 2, 3, + 2, 2, 2, 3, + 1, 2, 1, 2, 1, 2]; + assert.equal(tokens.count, lengths.length); + for (let i = 0; i < tokens.count; i += 1) { + const t = tokens.getItemAt(i); + assert.equal(t.type, TokenType.Operator, `${t.type} at ${i} is not an operator`); + assert.equal(t.length, lengths[i], `Length ${t.length} at ${i} (text ${text.substr(t.start, t.length)}), expected ${lengths[i]}`); + } + }); }); diff --git a/src/test/markdown/restTextConverter.test.ts b/src/test/markdown/restTextConverter.test.ts index 9b43d4d57657..ee08a8a9f2d6 100644 --- a/src/test/markdown/restTextConverter.test.ts +++ b/src/test/markdown/restTextConverter.test.ts @@ -1,30 +1,13 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -import { expect } from 'chai'; import * as fs from 'fs-extra'; import * as path from 'path'; import { RestTextConverter } from '../../client/common/markdown/restTextConverter'; +import { compareFiles } from '../textUtils'; const srcPythoFilesPath = path.join(__dirname, '..', '..', '..', 'src', 'test', 'pythonFiles', 'markdown'); -function compareFiles(expectedContent: string, actualContent: string) { - const expectedLines = expectedContent.split(/\r?\n/); - const actualLines = actualContent.split(/\r?\n/); - - for (let i = 0; i < Math.min(expectedLines.length, actualLines.length); i += 1) { - const e = expectedLines[i]; - const a = actualLines[i]; - expect(e, `Difference at line ${i}`).to.be.equal(a); - } - - expect(actualLines.length, - expectedLines.length > actualLines.length - ? 'Actual contains more lines than expected' - : 'Expected contains more lines than the actual' - ).to.be.equal(expectedLines.length); -} - async function testConversion(fileName: string): Promise { const cvt = new RestTextConverter(); const file = path.join(srcPythoFilesPath, fileName); diff --git a/src/test/pythonFiles/autocomp/hoverTest.py b/src/test/pythonFiles/autocomp/hoverTest.py index cd16059499a5..0ff88d80dffc 100644 --- a/src/test/pythonFiles/autocomp/hoverTest.py +++ b/src/test/pythonFiles/autocomp/hoverTest.py @@ -1,11 +1,11 @@ import random import math -for x in range(0,10): +for x in range(0, 10): print(x) rnd = random.Random() -print(rnd.randint(0,5)) +print(rnd.randint(0, 5)) print(math.acos(90)) import misc diff --git a/src/test/textUtils.ts b/src/test/textUtils.ts index b5d31b6e43f1..3805ab911dfd 100644 --- a/src/test/textUtils.ts +++ b/src/test/textUtils.ts @@ -1,5 +1,26 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import { expect } from 'chai'; import { MarkedString } from 'vscode'; export function normalizeMarkedString(content: MarkedString): string { return typeof content === 'string' ? content : content.value; } + +export function compareFiles(expectedContent: string, actualContent: string) { + const expectedLines = expectedContent.split(/\r?\n/); + const actualLines = actualContent.split(/\r?\n/); + + for (let i = 0; i < Math.min(expectedLines.length, actualLines.length); i += 1) { + const e = expectedLines[i]; + const a = actualLines[i]; + expect(e, `Difference at line ${i}`).to.be.equal(a); + } + + expect(actualLines.length, + expectedLines.length > actualLines.length + ? 'Actual contains more lines than expected' + : 'Expected contains more lines than the actual' + ).to.be.equal(expectedLines.length); +}