diff --git a/mypy/build.py b/mypy/build.py index d258256d156e..748735b9dd38 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -10,7 +10,6 @@ """ # TODO: More consistent terminology, e.g. path/fnam, module/id, state/file -import ast import binascii import collections import contextlib @@ -19,15 +18,13 @@ import gc import hashlib import json -import os.path +import os import pathlib import re import site import stat -import subprocess import sys import time -from os.path import dirname import errno from typing import (AbstractSet, Any, cast, Dict, Iterable, Iterator, List, @@ -37,7 +34,6 @@ from typing import ClassVar from typing_extensions import Final -from mypy import sitepkgs from mypy.nodes import (MypyFile, ImportBase, Import, ImportFrom, ImportAll) from mypy.semanal_pass1 import SemanticAnalyzerPass1 from mypy.semanal import SemanticAnalyzerPass2, apply_semantic_analyzer_patches @@ -49,6 +45,7 @@ from mypy.report import Reports from mypy import moduleinfo from mypy.fixup import fixup_module +from mypy.modulefinder import BuildSource, compute_search_paths, FindModuleCache, SearchPaths from mypy.nodes import Expression from mypy.options import Options from mypy.parse import parse @@ -71,9 +68,6 @@ DEBUG_FINE_GRAINED = False # type: Final -PYTHON_EXTENSIONS = ['.pyi', '.py'] # type: Final - - Graph = Dict[str, 'State'] @@ -98,20 +92,6 @@ def __init__(self, manager: 'BuildManager', graph: Graph) -> None: self.errors = [] # type: List[str] # Filled in by build if desired -class BuildSource: - def __init__(self, path: Optional[str], module: Optional[str], - text: Optional[str], base_dir: Optional[str] = None) -> None: - self.path = path - self.module = module or '__main__' - self.text = text - self.base_dir = base_dir - - def __repr__(self) -> str: - return '' % (self.path, - self.module, - self.text is not None) - - class BuildSourceSet: """Efficiently test a file's membership in the set of build sources.""" @@ -195,129 +175,6 @@ def default_flush_errors(new_messages: List[str], is_serious: bool) -> None: raise -# python_path is user code, mypy_path is set via config or environment variable, -# package_path is calculated by _get_site_packages_dirs, and typeshed_path points -# to typeshed. Each is a tuple of paths to be searched in find_module() -SearchPaths = NamedTuple('SearchPaths', - (('python_path', Tuple[str, ...]), - ('mypy_path', Tuple[str, ...]), - ('package_path', Tuple[str, ...]), - ('typeshed_path', Tuple[str, ...]))) - - -@functools.lru_cache(maxsize=None) -def _get_site_packages_dirs(python_executable: Optional[str], - fscache: FileSystemCache) -> Tuple[List[str], List[str]]: - """Find package directories for given python. - - This runs a subprocess call, which generates a list of the egg directories, and the site - package directories. To avoid repeatedly calling a subprocess (which can be slow!) we - lru_cache the results.""" - def make_abspath(path: str, root: str) -> str: - """Take a path and make it absolute relative to root if not already absolute.""" - if os.path.isabs(path): - return os.path.normpath(path) - else: - return os.path.join(root, os.path.normpath(path)) - - if python_executable is None: - return [], [] - if python_executable == sys.executable: - # Use running Python's package dirs - site_packages = sitepkgs.getsitepackages() - else: - # Use subprocess to get the package directory of given Python - # executable - site_packages = ast.literal_eval( - subprocess.check_output([python_executable, sitepkgs.__file__], - stderr=subprocess.PIPE).decode()) - egg_dirs = [] - for dir in site_packages: - pth = os.path.join(dir, 'easy-install.pth') - if fscache.isfile(pth): - with open(pth) as f: - egg_dirs.extend([make_abspath(d.rstrip(), dir) for d in f.readlines()]) - return egg_dirs, site_packages - - -def compute_search_paths(sources: List[BuildSource], - options: Options, - data_dir: str, - fscache: FileSystemCache, - alt_lib_path: Optional[str] = None) -> SearchPaths: - """Compute the search paths as specified in PEP 561. - - There are the following 4 members created: - - User code (from `sources`) - - MYPYPATH (set either via config or environment variable) - - installed package directories (which will later be split into stub-only and inline) - - typeshed - """ - # Determine the default module search path. - lib_path = collections.deque( - default_lib_path(data_dir, - options.python_version, - custom_typeshed_dir=options.custom_typeshed_dir)) - - if options.use_builtins_fixtures: - # Use stub builtins (to speed up test cases and to make them easier to - # debug). This is a test-only feature, so assume our files are laid out - # as in the source tree. - root_dir = dirname(dirname(__file__)) - lib_path.appendleft(os.path.join(root_dir, 'test-data', 'unit', 'lib-stub')) - # alt_lib_path is used by some tests to bypass the normal lib_path mechanics. - # If we don't have one, grab directories of source files. - python_path = [] # type: List[str] - if not alt_lib_path: - for source in sources: - # Include directory of the program file in the module search path. - if source.base_dir: - dir = source.base_dir - if dir not in python_path: - python_path.append(dir) - - # Do this even if running as a file, for sanity (mainly because with - # multiple builds, there could be a mix of files/modules, so its easier - # to just define the semantics that we always add the current director - # to the lib_path - # TODO: Don't do this in some cases; for motivation see see - # https://github.com/python/mypy/issues/4195#issuecomment-341915031 - if options.bazel: - dir = '.' - else: - dir = os.getcwd() - if dir not in lib_path: - python_path.insert(0, dir) - - # Start with a MYPYPATH environment variable at the front of the mypy_path, if defined. - mypypath = mypy_path() - - # Add a config-defined mypy path. - mypypath.extend(options.mypy_path) - - # If provided, insert the caller-supplied extra module path to the - # beginning (highest priority) of the search path. - if alt_lib_path: - mypypath.insert(0, alt_lib_path) - - egg_dirs, site_packages = _get_site_packages_dirs(options.python_executable, fscache) - for site_dir in site_packages: - assert site_dir not in lib_path - if site_dir in mypypath: - print("{} is in the MYPYPATH. Please remove it.".format(site_dir), file=sys.stderr) - sys.exit(1) - elif site_dir in python_path: - print("{} is in the PYTHONPATH. Please change directory" - " so it is not.".format(site_dir), - file=sys.stderr) - sys.exit(1) - - return SearchPaths(tuple(reversed(python_path)), - tuple(mypypath), - tuple(egg_dirs + site_packages), - tuple(lib_path)) - - def _build(sources: List[BuildSource], options: Options, alt_lib_path: Optional[str], @@ -372,54 +229,6 @@ def default_data_dir() -> str: return os.path.dirname(__file__) -def mypy_path() -> List[str]: - path_env = os.getenv('MYPYPATH') - if not path_env: - return [] - return path_env.split(os.pathsep) - - -def default_lib_path(data_dir: str, - pyversion: Tuple[int, int], - custom_typeshed_dir: Optional[str]) -> List[str]: - """Return default standard library search paths.""" - # IDEA: Make this more portable. - path = [] # type: List[str] - - if custom_typeshed_dir: - typeshed_dir = custom_typeshed_dir - else: - auto = os.path.join(data_dir, 'stubs-auto') - if os.path.isdir(auto): - data_dir = auto - typeshed_dir = os.path.join(data_dir, "typeshed") - if pyversion[0] == 3: - # We allow a module for e.g. version 3.5 to be in 3.4/. The assumption - # is that a module added with 3.4 will still be present in Python 3.5. - versions = ["%d.%d" % (pyversion[0], minor) - for minor in reversed(range(PYTHON3_VERSION_MIN[1], pyversion[1] + 1))] - else: - # For Python 2, we only have stubs for 2.7 - versions = ["2.7"] - # E.g. for Python 3.6, try 3.6/, 3.5/, 3.4/, 3/, 2and3/. - for v in versions + [str(pyversion[0]), '2and3']: - for lib_type in ['stdlib', 'third_party']: - stubdir = os.path.join(typeshed_dir, lib_type, v) - if os.path.isdir(stubdir): - path.append(stubdir) - - # Add fallback path that can be used if we have a broken installation. - if sys.platform != 'win32': - path.append('/usr/local/lib/mypy') - if not path: - print("Could not resolve typeshed subdirectories. If you are using mypy\n" - "from source, you need to run \"git submodule update --init\".\n" - "Otherwise your mypy install is broken.\nPython executable is located at " - "{0}.\nMypy located at {1}".format(sys.executable, data_dir), file=sys.stderr) - sys.exit(1) - return path - - CacheMeta = NamedTuple('CacheMeta', [('id', str), ('path', str), @@ -677,7 +486,7 @@ def __init__(self, data_dir: str, self.cache_enabled = options.incremental and ( not options.fine_grained_incremental or options.use_fine_grained_cache) self.fscache = fscache - self.find_module_cache = FindModuleCache(self.fscache, self.options) + self.find_module_cache = FindModuleCache(self.search_paths, self.fscache, self.options) # a mapping from source files to their corresponding shadow files # for efficient lookup @@ -807,8 +616,7 @@ def correct_rel_imp(imp: Union[ImportFrom, ImportAll]) -> str: def is_module(self, id: str) -> bool: """Is there a file in the file system corresponding to module id?""" - return self.find_module_cache.find_module(id, self.search_paths, - self.options.python_executable) is not None + return self.find_module_cache.find_module(id) is not None def parse_file(self, id: str, path: str, source: str, ignore_errors: bool) -> MypyFile: """Parse the source of a file with the given name. @@ -840,197 +648,6 @@ def stats_summary(self) -> Mapping[str, object]: return self.stats -# Package dirs are a two-tuple of path to search and whether to verify the module -OnePackageDir = Tuple[str, bool] -PackageDirs = List[OnePackageDir] - - -class FindModuleCache: - """Module finder with integrated cache. - - Module locations and some intermediate results are cached internally - and can be cleared with the clear() method. - - All file system accesses are performed through a FileSystemCache, - which is not ever cleared by this class. If necessary it must be - cleared by client code. - """ - - def __init__(self, fscache: Optional[FileSystemCache] = None, - options: Optional[Options] = None) -> None: - self.fscache = fscache or FileSystemCache() - # Cache find_lib_path_dirs: (dir_chain, search_paths) -> list(package_dirs, should_verify) - self.dirs = {} # type: Dict[Tuple[str, Tuple[str, ...]], PackageDirs] - # Cache find_module: (id, search_paths, python_version) -> result. - self.results = {} # type: Dict[Tuple[str, SearchPaths, Optional[str]], Optional[str]] - self.options = options - - def clear(self) -> None: - self.results.clear() - self.dirs.clear() - - def find_lib_path_dirs(self, dir_chain: str, lib_path: Tuple[str, ...]) -> PackageDirs: - # Cache some repeated work within distinct find_module calls: finding which - # elements of lib_path have even the subdirectory they'd need for the module - # to exist. This is shared among different module ids when they differ only - # in the last component. - # This is run for the python_path, mypy_path, and typeshed_path search paths - key = (dir_chain, lib_path) - if key not in self.dirs: - self.dirs[key] = self._find_lib_path_dirs(dir_chain, lib_path) - return self.dirs[key] - - def _find_lib_path_dirs(self, dir_chain: str, lib_path: Tuple[str, ...]) -> PackageDirs: - dirs = [] - for pathitem in lib_path: - # e.g., '/usr/lib/python3.4/foo/bar' - dir = os.path.normpath(os.path.join(pathitem, dir_chain)) - if self.fscache.isdir(dir): - dirs.append((dir, True)) - return dirs - - def find_module(self, id: str, search_paths: SearchPaths, - python_executable: Optional[str]) -> Optional[str]: - """Return the path of the module source file, or None if not found.""" - key = (id, search_paths, python_executable) - if key not in self.results: - self.results[key] = self._find_module(id, search_paths, python_executable) - return self.results[key] - - def _find_module_non_stub_helper(self, components: List[str], - pkg_dir: str) -> Optional[OnePackageDir]: - dir_path = pkg_dir - for index, component in enumerate(components): - dir_path = os.path.join(dir_path, component) - if self.fscache.isfile(os.path.join(dir_path, 'py.typed')): - return os.path.join(pkg_dir, *components[:-1]), index == 0 - return None - - def _find_module(self, id: str, search_paths: SearchPaths, - python_executable: Optional[str]) -> Optional[str]: - fscache = self.fscache - - # If we're looking for a module like 'foo.bar.baz', it's likely that most of the - # many elements of lib_path don't even have a subdirectory 'foo/bar'. Discover - # that only once and cache it for when we look for modules like 'foo.bar.blah' - # that will require the same subdirectory. - components = id.split('.') - dir_chain = os.sep.join(components[:-1]) # e.g., 'foo/bar' - # TODO (ethanhs): refactor each path search to its own method with lru_cache - - # We have two sets of folders so that we collect *all* stubs folders and - # put them in the front of the search path - third_party_inline_dirs = [] # type: PackageDirs - third_party_stubs_dirs = [] # type: PackageDirs - # Third-party stub/typed packages - for pkg_dir in search_paths.package_path: - stub_name = components[0] + '-stubs' - stub_dir = os.path.join(pkg_dir, stub_name) - if fscache.isdir(stub_dir): - stub_typed_file = os.path.join(stub_dir, 'py.typed') - stub_components = [stub_name] + components[1:] - path = os.path.join(pkg_dir, *stub_components[:-1]) - if fscache.isdir(path): - if fscache.isfile(stub_typed_file): - # Stub packages can have a py.typed file, which must include - # 'partial\n' to make the package partial - # Partial here means that mypy should look at the runtime - # package if installed. - if fscache.read(stub_typed_file).decode().strip() == 'partial': - runtime_path = os.path.join(pkg_dir, dir_chain) - third_party_inline_dirs.append((runtime_path, True)) - # if the package is partial, we don't verify the module, as - # the partial stub package may not have a __init__.pyi - third_party_stubs_dirs.append((path, False)) - else: - third_party_stubs_dirs.append((path, True)) - non_stub_match = self._find_module_non_stub_helper(components, pkg_dir) - if non_stub_match: - third_party_inline_dirs.append(non_stub_match) - if self.options and self.options.use_builtins_fixtures: - # Everything should be in fixtures. - third_party_inline_dirs.clear() - third_party_stubs_dirs.clear() - python_mypy_path = search_paths.python_path + search_paths.mypy_path - candidate_base_dirs = self.find_lib_path_dirs(dir_chain, python_mypy_path) + \ - third_party_stubs_dirs + third_party_inline_dirs + \ - self.find_lib_path_dirs(dir_chain, search_paths.typeshed_path) - - # If we're looking for a module like 'foo.bar.baz', then candidate_base_dirs now - # contains just the subdirectories 'foo/bar' that actually exist under the - # elements of lib_path. This is probably much shorter than lib_path itself. - # Now just look for 'baz.pyi', 'baz/__init__.py', etc., inside those directories. - seplast = os.sep + components[-1] # so e.g. '/baz' - sepinit = os.sep + '__init__' - for base_dir, verify in candidate_base_dirs: - base_path = base_dir + seplast # so e.g. '/usr/lib/python3.4/foo/bar/baz' - # Prefer package over module, i.e. baz/__init__.py* over baz.py*. - for extension in PYTHON_EXTENSIONS: - path = base_path + sepinit + extension - path_stubs = base_path + '-stubs' + sepinit + extension - if fscache.isfile_case(path): - if verify and not verify_module(fscache, id, path): - continue - return path - elif fscache.isfile_case(path_stubs): - if verify and not verify_module(fscache, id, path_stubs): - continue - return path_stubs - # No package, look for module. - for extension in PYTHON_EXTENSIONS: - path = base_path + extension - if fscache.isfile_case(path): - if verify and not verify_module(fscache, id, path): - continue - return path - return None - - def find_modules_recursive(self, module: str, search_paths: SearchPaths, - python_executable: Optional[str]) -> List[BuildSource]: - module_path = self.find_module(module, search_paths, python_executable) - if not module_path: - return [] - result = [BuildSource(module_path, module, None)] - if module_path.endswith(('__init__.py', '__init__.pyi')): - # Subtle: this code prefers the .pyi over the .py if both - # exists, and also prefers packages over modules if both x/ - # and x.py* exist. How? We sort the directory items, so x - # comes before x.py and x.pyi. But the preference for .pyi - # over .py is encoded in find_module(); even though we see - # x.py before x.pyi, find_module() will find x.pyi first. We - # use hits to avoid adding it a second time when we see x.pyi. - # This also avoids both x.py and x.pyi when x/ was seen first. - hits = set() # type: Set[str] - for item in sorted(self.fscache.listdir(os.path.dirname(module_path))): - abs_path = os.path.join(os.path.dirname(module_path), item) - if os.path.isdir(abs_path) and \ - (os.path.isfile(os.path.join(abs_path, '__init__.py')) or - os.path.isfile(os.path.join(abs_path, '__init__.pyi'))): - hits.add(item) - result += self.find_modules_recursive(module + '.' + item, search_paths, - python_executable) - elif item != '__init__.py' and item != '__init__.pyi' and \ - item.endswith(('.py', '.pyi')): - mod = item.split('.')[0] - if mod not in hits: - hits.add(mod) - result += self.find_modules_recursive(module + '.' + mod, search_paths, - python_executable) - return result - - -def verify_module(fscache: FileSystemCache, id: str, path: str) -> bool: - """Check that all packages containing id have a __init__ file.""" - if path.endswith(('__init__.py', '__init__.pyi')): - path = dirname(path) - for i in range(id.count('.')): - path = dirname(path) - if not any(fscache.isfile_case(os.path.join(path, '__init__{}'.format(extension))) - for extension in PYTHON_EXTENSIONS): - return False - return True - - def write_protocol_deps_cache(proto_deps: Dict[str, Set[str]], manager: BuildManager, graph: Graph) -> None: """Write cache files for protocol dependencies. @@ -2321,8 +1938,7 @@ def find_module_and_diagnose(manager: BuildManager, # difference and just assume 'builtins' everywhere, # which simplifies code. file_id = '__builtin__' - path = manager.find_module_cache.find_module(file_id, manager.search_paths, - manager.options.python_executable) + path = manager.find_module_cache.find_module(file_id) if path: # For non-stubs, look at options.follow_imports: # - normal (default) -> fully analyze @@ -2401,8 +2017,7 @@ def exist_added_packages(suppressed: List[str], def find_module_simple(id: str, manager: BuildManager) -> Optional[str]: """Find a filesystem path for module `id` or `None` if not found.""" - return manager.find_module_cache.find_module(id, manager.search_paths, - manager.options.python_executable) + return manager.find_module_cache.find_module(id) def in_partial_package(id: str, manager: BuildManager) -> bool: diff --git a/mypy/find_sources.py b/mypy/find_sources.py index 0ee03778a829..cdf586851206 100644 --- a/mypy/find_sources.py +++ b/mypy/find_sources.py @@ -4,7 +4,7 @@ from typing import List, Sequence, Set, Tuple, Optional, Dict -from mypy.build import BuildSource, PYTHON_EXTENSIONS +from mypy.modulefinder import BuildSource, PYTHON_EXTENSIONS from mypy.fscache import FileSystemCache from mypy.options import Options diff --git a/mypy/main.py b/mypy/main.py index 00cb1987a761..8cf5e9703b3b 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -15,7 +15,8 @@ from mypy import defaults from mypy import experiments from mypy import util -from mypy.build import BuildSource, BuildResult, SearchPaths +from mypy.build import BuildResult +from mypy.modulefinder import BuildSource, FindModuleCache, mypy_path, SearchPaths from mypy.find_sources import create_source_list, InvalidSourceList from mypy.fscache import FileSystemCache from mypy.errors import CompileError @@ -453,6 +454,10 @@ def add_invertible_flag(flag: str, imports_group.add_argument( '--no-silence-site-packages', action='store_true', help="Do not silence errors in PEP 561 compliant installed packages") + add_invertible_flag( + '--namespace-packages', default=False, + help="Support namespace packages (PEP 420, __init__.py-less)", + group=imports_group) platform_group = parser.add_argument_group( title='Platform configuration', @@ -876,14 +881,14 @@ def add_invertible_flag(flag: str, # Set target. if special_opts.modules + special_opts.packages: options.build_type = BuildType.MODULE - search_paths = SearchPaths((os.getcwd(),), tuple(build.mypy_path()), (), ()) + search_paths = SearchPaths((os.getcwd(),), tuple(mypy_path()), (), ()) targets = [] # TODO: use the same cache that the BuildManager will - cache = build.FindModuleCache(fscache) + cache = FindModuleCache(search_paths, fscache) for p in special_opts.packages: if os.sep in p or os.altsep and os.altsep in p: fail("Package name '{}' cannot have a slash in it.".format(p)) - p_targets = cache.find_modules_recursive(p, search_paths, options.python_executable) + p_targets = cache.find_modules_recursive(p) if not p_targets: fail("Can't find package '{}'".format(p)) targets.extend(p_targets) diff --git a/mypy/modulefinder.py b/mypy/modulefinder.py new file mode 100644 index 000000000000..d98bea3c2c83 --- /dev/null +++ b/mypy/modulefinder.py @@ -0,0 +1,397 @@ +"""Low-level infrastructure to find modules. + +This build on fscache.py; find_sources.py builds on top of this. +""" + +import ast +import collections +import functools +import os +import subprocess +import sys + +from typing import Dict, List, NamedTuple, Optional, Set, Tuple + +MYPY = False +if MYPY: + from typing_extensions import Final + +from mypy.defaults import PYTHON3_VERSION_MIN +from mypy.fscache import FileSystemCache +from mypy.options import Options +from mypy import sitepkgs + +# Paths to be searched in find_module(). +SearchPaths = NamedTuple( + 'SearchPaths', + [('python_path', Tuple[str, ...]), # where user code is found + ('mypy_path', Tuple[str, ...]), # from $MYPYPATH or config variable + ('package_path', Tuple[str, ...]), # from get_site_packages_dirs() + ('typeshed_path', Tuple[str, ...]), # paths in typeshed + ]) + +# Package dirs are a two-tuple of path to search and whether to verify the module +OnePackageDir = Tuple[str, bool] +PackageDirs = List[OnePackageDir] + +PYTHON_EXTENSIONS = ['.pyi', '.py'] # type: Final + + +class BuildSource: + """A single source file.""" + + def __init__(self, path: Optional[str], module: Optional[str], + text: Optional[str], base_dir: Optional[str] = None) -> None: + self.path = path # File where it's found (e.g. 'xxx/yyy/foo/bar.py') + self.module = module or '__main__' # Module name (e.g. 'foo.bar') + self.text = text # Source code, if initially supplied, else None + self.base_dir = base_dir # Directory where the package is rooted (e.g. 'xxx/yyy') + + def __repr__(self) -> str: + return '' % (self.path, + self.module, + self.text is not None) + + +class FindModuleCache: + """Module finder with integrated cache. + + Module locations and some intermediate results are cached internally + and can be cleared with the clear() method. + + All file system accesses are performed through a FileSystemCache, + which is not ever cleared by this class. If necessary it must be + cleared by client code. + """ + + def __init__(self, + search_paths: SearchPaths, + fscache: Optional[FileSystemCache] = None, + options: Optional[Options] = None) -> None: + self.search_paths = search_paths + self.fscache = fscache or FileSystemCache() + # Cache find_lib_path_dirs: (dir_chain, search_paths) -> list(package_dirs, should_verify) + self.dirs = {} # type: Dict[Tuple[str, Tuple[str, ...]], PackageDirs] + # Cache find_module: id -> result + self.results = {} # type: Dict[str, Optional[str]] + self.options = options + + def clear(self) -> None: + self.results.clear() + self.dirs.clear() + + def find_lib_path_dirs(self, dir_chain: str, lib_path: Tuple[str, ...]) -> PackageDirs: + # Cache some repeated work within distinct find_module calls: finding which + # elements of lib_path have even the subdirectory they'd need for the module + # to exist. This is shared among different module ids when they differ only + # in the last component. + # This is run for the python_path, mypy_path, and typeshed_path search paths + key = (dir_chain, lib_path) + if key not in self.dirs: + self.dirs[key] = self._find_lib_path_dirs(dir_chain, lib_path) + return self.dirs[key] + + def _find_lib_path_dirs(self, dir_chain: str, lib_path: Tuple[str, ...]) -> PackageDirs: + dirs = [] + for pathitem in lib_path: + # e.g., '/usr/lib/python3.4/foo/bar' + dir = os.path.normpath(os.path.join(pathitem, dir_chain)) + if self.fscache.isdir(dir): + dirs.append((dir, True)) + return dirs + + def find_module(self, id: str) -> Optional[str]: + """Return the path of the module source file, or None if not found.""" + if id not in self.results: + self.results[id] = self._find_module(id) + return self.results[id] + + def _find_module_non_stub_helper(self, components: List[str], + pkg_dir: str) -> Optional[OnePackageDir]: + dir_path = pkg_dir + for index, component in enumerate(components): + dir_path = os.path.join(dir_path, component) + if self.fscache.isfile(os.path.join(dir_path, 'py.typed')): + return os.path.join(pkg_dir, *components[:-1]), index == 0 + return None + + def _find_module(self, id: str) -> Optional[str]: + fscache = self.fscache + + # If we're looking for a module like 'foo.bar.baz', it's likely that most of the + # many elements of lib_path don't even have a subdirectory 'foo/bar'. Discover + # that only once and cache it for when we look for modules like 'foo.bar.blah' + # that will require the same subdirectory. + components = id.split('.') + dir_chain = os.sep.join(components[:-1]) # e.g., 'foo/bar' + # TODO (ethanhs): refactor each path search to its own method with lru_cache + + # We have two sets of folders so that we collect *all* stubs folders and + # put them in the front of the search path + third_party_inline_dirs = [] # type: PackageDirs + third_party_stubs_dirs = [] # type: PackageDirs + # Third-party stub/typed packages + for pkg_dir in self.search_paths.package_path: + stub_name = components[0] + '-stubs' + stub_dir = os.path.join(pkg_dir, stub_name) + if fscache.isdir(stub_dir): + stub_typed_file = os.path.join(stub_dir, 'py.typed') + stub_components = [stub_name] + components[1:] + path = os.path.join(pkg_dir, *stub_components[:-1]) + if fscache.isdir(path): + if fscache.isfile(stub_typed_file): + # Stub packages can have a py.typed file, which must include + # 'partial\n' to make the package partial + # Partial here means that mypy should look at the runtime + # package if installed. + if fscache.read(stub_typed_file).decode().strip() == 'partial': + runtime_path = os.path.join(pkg_dir, dir_chain) + third_party_inline_dirs.append((runtime_path, True)) + # if the package is partial, we don't verify the module, as + # the partial stub package may not have a __init__.pyi + third_party_stubs_dirs.append((path, False)) + else: + third_party_stubs_dirs.append((path, True)) + non_stub_match = self._find_module_non_stub_helper(components, pkg_dir) + if non_stub_match: + third_party_inline_dirs.append(non_stub_match) + if self.options and self.options.use_builtins_fixtures: + # Everything should be in fixtures. + third_party_inline_dirs.clear() + third_party_stubs_dirs.clear() + python_mypy_path = self.search_paths.python_path + self.search_paths.mypy_path + candidate_base_dirs = self.find_lib_path_dirs(dir_chain, python_mypy_path) + \ + third_party_stubs_dirs + third_party_inline_dirs + \ + self.find_lib_path_dirs(dir_chain, self.search_paths.typeshed_path) + + # If we're looking for a module like 'foo.bar.baz', then candidate_base_dirs now + # contains just the subdirectories 'foo/bar' that actually exist under the + # elements of lib_path. This is probably much shorter than lib_path itself. + # Now just look for 'baz.pyi', 'baz/__init__.py', etc., inside those directories. + seplast = os.sep + components[-1] # so e.g. '/baz' + sepinit = os.sep + '__init__' + for base_dir, verify in candidate_base_dirs: + base_path = base_dir + seplast # so e.g. '/usr/lib/python3.4/foo/bar/baz' + # Prefer package over module, i.e. baz/__init__.py* over baz.py*. + for extension in PYTHON_EXTENSIONS: + path = base_path + sepinit + extension + path_stubs = base_path + '-stubs' + sepinit + extension + if fscache.isfile_case(path): + if verify and not verify_module(fscache, id, path): + continue + return path + elif fscache.isfile_case(path_stubs): + if verify and not verify_module(fscache, id, path_stubs): + continue + return path_stubs + # No package, look for module. + for extension in PYTHON_EXTENSIONS: + path = base_path + extension + if fscache.isfile_case(path): + if verify and not verify_module(fscache, id, path): + continue + return path + return None + + def find_modules_recursive(self, module: str) -> List[BuildSource]: + module_path = self.find_module(module) + if not module_path: + return [] + result = [BuildSource(module_path, module, None)] + if module_path.endswith(('__init__.py', '__init__.pyi')): + # Subtle: this code prefers the .pyi over the .py if both + # exists, and also prefers packages over modules if both x/ + # and x.py* exist. How? We sort the directory items, so x + # comes before x.py and x.pyi. But the preference for .pyi + # over .py is encoded in find_module(); even though we see + # x.py before x.pyi, find_module() will find x.pyi first. We + # use hits to avoid adding it a second time when we see x.pyi. + # This also avoids both x.py and x.pyi when x/ was seen first. + hits = set() # type: Set[str] + for item in sorted(self.fscache.listdir(os.path.dirname(module_path))): + abs_path = os.path.join(os.path.dirname(module_path), item) + if os.path.isdir(abs_path) and \ + (os.path.isfile(os.path.join(abs_path, '__init__.py')) or + os.path.isfile(os.path.join(abs_path, '__init__.pyi'))): + hits.add(item) + result += self.find_modules_recursive(module + '.' + item) + elif item != '__init__.py' and item != '__init__.pyi' and \ + item.endswith(('.py', '.pyi')): + mod = item.split('.')[0] + if mod not in hits: + hits.add(mod) + result += self.find_modules_recursive(module + '.' + mod) + return result + + +def verify_module(fscache: FileSystemCache, id: str, path: str) -> bool: + """Check that all packages containing id have a __init__ file.""" + if path.endswith(('__init__.py', '__init__.pyi')): + path = os.path.dirname(path) + for i in range(id.count('.')): + path = os.path.dirname(path) + if not any(fscache.isfile_case(os.path.join(path, '__init__{}'.format(extension))) + for extension in PYTHON_EXTENSIONS): + return False + return True + + +def mypy_path() -> List[str]: + path_env = os.getenv('MYPYPATH') + if not path_env: + return [] + return path_env.split(os.pathsep) + + +def default_lib_path(data_dir: str, + pyversion: Tuple[int, int], + custom_typeshed_dir: Optional[str]) -> List[str]: + """Return default standard library search paths.""" + # IDEA: Make this more portable. + path = [] # type: List[str] + + if custom_typeshed_dir: + typeshed_dir = custom_typeshed_dir + else: + auto = os.path.join(data_dir, 'stubs-auto') + if os.path.isdir(auto): + data_dir = auto + typeshed_dir = os.path.join(data_dir, "typeshed") + if pyversion[0] == 3: + # We allow a module for e.g. version 3.5 to be in 3.4/. The assumption + # is that a module added with 3.4 will still be present in Python 3.5. + versions = ["%d.%d" % (pyversion[0], minor) + for minor in reversed(range(PYTHON3_VERSION_MIN[1], pyversion[1] + 1))] + else: + # For Python 2, we only have stubs for 2.7 + versions = ["2.7"] + # E.g. for Python 3.6, try 3.6/, 3.5/, 3.4/, 3/, 2and3/. + for v in versions + [str(pyversion[0]), '2and3']: + for lib_type in ['stdlib', 'third_party']: + stubdir = os.path.join(typeshed_dir, lib_type, v) + if os.path.isdir(stubdir): + path.append(stubdir) + + # Add fallback path that can be used if we have a broken installation. + if sys.platform != 'win32': + path.append('/usr/local/lib/mypy') + if not path: + print("Could not resolve typeshed subdirectories. If you are using mypy\n" + "from source, you need to run \"git submodule update --init\".\n" + "Otherwise your mypy install is broken.\nPython executable is located at " + "{0}.\nMypy located at {1}".format(sys.executable, data_dir), file=sys.stderr) + sys.exit(1) + return path + + +@functools.lru_cache(maxsize=None) +def get_site_packages_dirs(python_executable: Optional[str], + fscache: FileSystemCache) -> Tuple[List[str], List[str]]: + """Find package directories for given python. + + This runs a subprocess call, which generates a list of the egg directories, and the site + package directories. To avoid repeatedly calling a subprocess (which can be slow!) we + lru_cache the results.""" + def make_abspath(path: str, root: str) -> str: + """Take a path and make it absolute relative to root if not already absolute.""" + if os.path.isabs(path): + return os.path.normpath(path) + else: + return os.path.join(root, os.path.normpath(path)) + + if python_executable is None: + return [], [] + if python_executable == sys.executable: + # Use running Python's package dirs + site_packages = sitepkgs.getsitepackages() + else: + # Use subprocess to get the package directory of given Python + # executable + site_packages = ast.literal_eval( + subprocess.check_output([python_executable, sitepkgs.__file__], + stderr=subprocess.PIPE).decode()) + egg_dirs = [] + for dir in site_packages: + pth = os.path.join(dir, 'easy-install.pth') + if fscache.isfile(pth): + with open(pth) as f: + egg_dirs.extend([make_abspath(d.rstrip(), dir) for d in f.readlines()]) + return egg_dirs, site_packages + + +def compute_search_paths(sources: List[BuildSource], + options: Options, + data_dir: str, + fscache: FileSystemCache, + alt_lib_path: Optional[str] = None) -> SearchPaths: + """Compute the search paths as specified in PEP 561. + + There are the following 4 members created: + - User code (from `sources`) + - MYPYPATH (set either via config or environment variable) + - installed package directories (which will later be split into stub-only and inline) + - typeshed + """ + # Determine the default module search path. + lib_path = collections.deque( + default_lib_path(data_dir, + options.python_version, + custom_typeshed_dir=options.custom_typeshed_dir)) + + if options.use_builtins_fixtures: + # Use stub builtins (to speed up test cases and to make them easier to + # debug). This is a test-only feature, so assume our files are laid out + # as in the source tree. + root_dir = os.path.dirname(os.path.dirname(__file__)) + lib_path.appendleft(os.path.join(root_dir, 'test-data', 'unit', 'lib-stub')) + # alt_lib_path is used by some tests to bypass the normal lib_path mechanics. + # If we don't have one, grab directories of source files. + python_path = [] # type: List[str] + if not alt_lib_path: + for source in sources: + # Include directory of the program file in the module search path. + if source.base_dir: + dir = source.base_dir + if dir not in python_path: + python_path.append(dir) + + # Do this even if running as a file, for sanity (mainly because with + # multiple builds, there could be a mix of files/modules, so its easier + # to just define the semantics that we always add the current director + # to the lib_path + # TODO: Don't do this in some cases; for motivation see see + # https://github.com/python/mypy/issues/4195#issuecomment-341915031 + if options.bazel: + dir = '.' + else: + dir = os.getcwd() + if dir not in lib_path: + python_path.insert(0, dir) + + # Start with a MYPYPATH environment variable at the front of the mypy_path, if defined. + mypypath = mypy_path() + + # Add a config-defined mypy path. + mypypath.extend(options.mypy_path) + + # If provided, insert the caller-supplied extra module path to the + # beginning (highest priority) of the search path. + if alt_lib_path: + mypypath.insert(0, alt_lib_path) + + egg_dirs, site_packages = get_site_packages_dirs(options.python_executable, fscache) + for site_dir in site_packages: + assert site_dir not in lib_path + if site_dir in mypypath: + print("{} is in the MYPYPATH. Please remove it.".format(site_dir), file=sys.stderr) + sys.exit(1) + elif site_dir in python_path: + print("{} is in the PYTHONPATH. Please change directory" + " so it is not.".format(site_dir), + file=sys.stderr) + sys.exit(1) + + return SearchPaths(tuple(reversed(python_path)), + tuple(mypypath), + tuple(egg_dirs + site_packages), + tuple(lib_path)) diff --git a/mypy/options.py b/mypy/options.py index 066e60e73584..8ad73679878f 100644 --- a/mypy/options.py +++ b/mypy/options.py @@ -79,7 +79,9 @@ def __init__(self) -> None: self.follow_imports = 'normal' # normal|silent|skip|error # Whether to respect the follow_imports setting even for stub files. # Intended to be used for disabling specific stubs. - self.follow_imports_for_stubs = False # type: bool + self.follow_imports_for_stubs = False + # PEP 420 namespace packages + self.namespace_packages = False # disallow_any options self.disallow_any_generics = False diff --git a/mypy/stubgen.py b/mypy/stubgen.py index e6b8eb6ec317..5729e2ee9d63 100755 --- a/mypy/stubgen.py +++ b/mypy/stubgen.py @@ -57,6 +57,7 @@ import mypy.traverser import mypy.util from mypy import defaults +from mypy.modulefinder import FindModuleCache, SearchPaths from mypy.nodes import ( Expression, IntExpr, UnaryExpr, StrExpr, BytesExpr, NameExpr, FloatExpr, MemberExpr, TupleExpr, ListExpr, ComparisonExpr, CallExpr, IndexExpr, EllipsisExpr, @@ -165,9 +166,8 @@ def find_module_path_and_all(module: str, pyversion: Tuple[int, int], module_all = getattr(mod, '__all__', None) else: # Find module by going through search path. - search_paths = mypy.build.SearchPaths(('.',) + tuple(search_path), (), (), ()) - module_path = mypy.build.FindModuleCache().find_module(module, search_paths, - interpreter) + search_paths = SearchPaths(('.',) + tuple(search_path), (), (), ()) + module_path = FindModuleCache(search_paths).find_module(module) if not module_path: raise SystemExit( "Can't find module '{}' (consider using --search-path)".format(module)) diff --git a/mypy/test/testcheck.py b/mypy/test/testcheck.py index 5ed849f785be..7869ec07d7c3 100644 --- a/mypy/test/testcheck.py +++ b/mypy/test/testcheck.py @@ -7,7 +7,8 @@ from typing import Dict, List, Set, Tuple from mypy import build -from mypy.build import BuildSource, Graph, SearchPaths +from mypy.build import Graph +from mypy.modulefinder import BuildSource, SearchPaths from mypy.test.config import test_temp_dir, test_data_prefix from mypy.test.data import DataDrivenTestCase, DataSuite, FileOperation, UpdateFile from mypy.test.helpers import ( @@ -286,8 +287,7 @@ def parse_module(self, out = [] search_paths = SearchPaths((test_temp_dir,), (), (), ()) for module_name in module_names.split(' '): - path = build.FindModuleCache().find_module(module_name, search_paths, - sys.executable) + path = build.FindModuleCache(search_paths).find_module(module_name) assert path is not None, "Can't find ad hoc case file" with open(path) as f: program_text = f.read() diff --git a/mypy/test/testdeps.py b/mypy/test/testdeps.py index 5e1b012df986..34b37259ff90 100644 --- a/mypy/test/testdeps.py +++ b/mypy/test/testdeps.py @@ -8,7 +8,7 @@ from collections import defaultdict from mypy import build, defaults -from mypy.build import BuildSource +from mypy.modulefinder import BuildSource from mypy.errors import CompileError from mypy.nodes import MypyFile, Expression from mypy.options import Options diff --git a/mypy/test/testdiff.py b/mypy/test/testdiff.py index 499a12f16c0c..6e839b228b18 100644 --- a/mypy/test/testdiff.py +++ b/mypy/test/testdiff.py @@ -4,7 +4,7 @@ from typing import List, Tuple, Dict, Optional from mypy import build -from mypy.build import BuildSource +from mypy.modulefinder import BuildSource from mypy.defaults import PYTHON3_VERSION from mypy.errors import CompileError from mypy.nodes import MypyFile diff --git a/mypy/test/testerrorstream.py b/mypy/test/testerrorstream.py index 3df1f27eb60e..a9fbb95a7643 100644 --- a/mypy/test/testerrorstream.py +++ b/mypy/test/testerrorstream.py @@ -4,7 +4,7 @@ from mypy import build from mypy.test.helpers import assert_string_arrays_equal from mypy.test.data import DataDrivenTestCase, DataSuite -from mypy.build import BuildSource +from mypy.modulefinder import BuildSource from mypy.errors import CompileError from mypy.options import Options diff --git a/mypy/test/testfinegrained.py b/mypy/test/testfinegrained.py index 95561ab6efe2..83a1fa0f529b 100644 --- a/mypy/test/testfinegrained.py +++ b/mypy/test/testfinegrained.py @@ -18,7 +18,7 @@ from typing import List, cast from mypy import build -from mypy.build import BuildSource +from mypy.modulefinder import BuildSource from mypy.errors import CompileError from mypy.options import Options from mypy.test.config import test_temp_dir diff --git a/mypy/test/testgraph.py b/mypy/test/testgraph.py index ac88efe50ab5..81d70c5468f8 100644 --- a/mypy/test/testgraph.py +++ b/mypy/test/testgraph.py @@ -3,7 +3,8 @@ from typing import AbstractSet, Dict, Set, List from mypy.test.helpers import assert_equal, Suite -from mypy.build import BuildManager, State, BuildSourceSet, SearchPaths +from mypy.build import BuildManager, State, BuildSourceSet +from mypy.modulefinder import SearchPaths from mypy.build import topsort, strongly_connected_components, sorted_components, order_ascc from mypy.version import __version__ from mypy.options import Options diff --git a/mypy/test/testmerge.py b/mypy/test/testmerge.py index 8b010151fe7b..9a2799e7a845 100644 --- a/mypy/test/testmerge.py +++ b/mypy/test/testmerge.py @@ -5,7 +5,8 @@ from typing import List, Tuple, Dict, Optional from mypy import build -from mypy.build import BuildSource, BuildResult +from mypy.build import BuildResult +from mypy.modulefinder import BuildSource from mypy.defaults import PYTHON3_VERSION from mypy.errors import CompileError from mypy.nodes import ( diff --git a/mypy/test/testpep561.py b/mypy/test/testpep561.py index 066a15bca977..3cef3f66dfeb 100644 --- a/mypy/test/testpep561.py +++ b/mypy/test/testpep561.py @@ -6,7 +6,8 @@ from unittest import TestCase, main import mypy.api -from mypy.build import _get_site_packages_dirs, FileSystemCache +from mypy.build import FileSystemCache +from mypy.modulefinder import get_site_packages_dirs from mypy.test.config import package_path from mypy.test.helpers import run_command from mypy.util import try_find_python2_interpreter @@ -129,7 +130,7 @@ def tearDown(self) -> None: def test_get_pkg_dirs(self) -> None: """Check that get_package_dirs works.""" - dirs = _get_site_packages_dirs(sys.executable, FileSystemCache()) + dirs = get_site_packages_dirs(sys.executable, FileSystemCache()) assert dirs def test_typedpkg_stub_package(self) -> None: diff --git a/mypy/test/testsemanal.py b/mypy/test/testsemanal.py index c71e7a0ac4be..b07f31fbf5d9 100644 --- a/mypy/test/testsemanal.py +++ b/mypy/test/testsemanal.py @@ -5,7 +5,7 @@ from typing import Dict, List from mypy import build -from mypy.build import BuildSource +from mypy.modulefinder import BuildSource from mypy.defaults import PYTHON3_VERSION from mypy.test.helpers import ( assert_string_arrays_equal, normalize_error_messages, testfile_pyversion, diff --git a/mypy/test/testtransform.py b/mypy/test/testtransform.py index cc15a63ad844..9e22c15285c5 100644 --- a/mypy/test/testtransform.py +++ b/mypy/test/testtransform.py @@ -3,7 +3,7 @@ import os.path from mypy import build -from mypy.build import BuildSource +from mypy.modulefinder import BuildSource from mypy.test.helpers import ( assert_string_arrays_equal, testfile_pyversion, normalize_error_messages ) diff --git a/mypy/test/testtypegen.py b/mypy/test/testtypegen.py index 1baa9e9187cc..d3c178890b58 100644 --- a/mypy/test/testtypegen.py +++ b/mypy/test/testtypegen.py @@ -3,7 +3,7 @@ import re from mypy import build -from mypy.build import BuildSource +from mypy.modulefinder import BuildSource from mypy.test.config import test_temp_dir from mypy.test.data import DataDrivenTestCase, DataSuite from mypy.test.helpers import assert_string_arrays_equal