Merge remote-tracking branch 'upstream/master' into pytest-complete

Author: elazarg

docs/source/installed_packages.rst

@@ -43,7 +43,7 @@ If you would like to publish a library package to a package repository (e.g.
 PyPI) for either internal or external use in type checking, packages that
 supply type information via type comments or annotations in the code should put
 a ``py.typed`` in their package directory. For example, with a directory
-structure as follows:
+structure as follows
 
 .. code-block:: text
 
@@ -53,7 +53,7 @@ structure as follows:
         lib.py
         py.typed
 
-the setup.py might look like:
+the setup.py might look like
 
 .. code-block:: python
 
@@ -67,8 +67,13 @@ the setup.py might look like:
         packages=["package_a"]
     )
 
+.. note::
+
+   If you use setuptools, you must pass the option ``zip_safe=False`` to
+   ``setup()``, or mypy will not be able to find the installed package.
+
 Some packages have a mix of stub files and runtime files. These packages also
-require a ``py.typed`` file. An example can be seen below:
+require a ``py.typed`` file. An example can be seen below
 
 .. code-block:: text
 

extensions/mypy_extensions.py

@@ -135,3 +135,7 @@ def KwArg(type=Any):
 
 # Return type that indicates a function does not return
 class NoReturn: pass
+
+
+def trait(cls):
+    return cls

mypy/build.py

@@ -199,26 +199,43 @@ def default_flush_errors(new_messages: List[str], is_serious: bool) -> None:
 
 
 @functools.lru_cache(maxsize=None)
-def _get_site_packages_dirs(python_executable: Optional[str]) -> List[str]:
+def _get_site_packages_dirs(python_executable: Optional[str],
+                            fscache: FileSystemCache) -> List[str]:
     """Find package directories for given python.
 
     This runs a subprocess call, which generates a list of the site package directories.
     To avoid repeatedly calling a subprocess (which can be slow!) we lru_cache the results."""
+    def make_abspath(path: str, root: str) -> str:
+        """Take a path and make it absolute relative to root if not already absolute."""
+        if os.path.isabs(path):
+            return os.path.normpath(path)
+        else:
+            return os.path.join(root, os.path.normpath(path))
+
     if python_executable is None:
         return []
     if python_executable == sys.executable:
         # Use running Python's package dirs
-        return sitepkgs.getsitepackages()
+        site_packages = sitepkgs.getsitepackages()
     else:
         # Use subprocess to get the package directory of given Python
         # executable
-        return ast.literal_eval(subprocess.check_output([python_executable, sitepkgs.__file__],
-                                stderr=subprocess.PIPE).decode())
+        site_packages = ast.literal_eval(
+            subprocess.check_output([python_executable, sitepkgs.__file__],
+            stderr=subprocess.PIPE).decode())
+    egg_dirs = []
+    for dir in site_packages:
+        pth = os.path.join(dir, 'easy-install.pth')
+        if fscache.isfile(pth):
+            with open(pth) as f:
+                egg_dirs.extend([make_abspath(d.rstrip(), dir) for d in f.readlines()])
+    return egg_dirs + site_packages
 
 
 def compute_search_paths(sources: List[BuildSource],
                      options: Options,
                      data_dir: str,
+                     fscache: FileSystemCache,
                      alt_lib_path: Optional[str] = None) -> SearchPaths:
     """Compute the search paths as specified in PEP 561.
 
@@ -275,9 +292,21 @@ def compute_search_paths(sources: List[BuildSource],
     if alt_lib_path:
         mypypath.insert(0, alt_lib_path)
 
+    package_path = tuple(_get_site_packages_dirs(options.python_executable, fscache))
+    for site_dir in package_path:
+        assert site_dir not in lib_path
+        if site_dir in mypypath:
+            print("{} is in the MYPYPATH. Please remove it.".format(site_dir), file=sys.stderr)
+            sys.exit(1)
+        elif site_dir in python_path:
+            print("{} is in the PYTHONPATH. Please change directory"
+                  " so it is not.".format(site_dir),
+                  file=sys.stderr)
+            sys.exit(1)
+
     return SearchPaths(tuple(reversed(python_path)),
                        tuple(mypypath),
-                       tuple(_get_site_packages_dirs(options.python_executable)),
+                       package_path,
                        tuple(lib_path))
 
 
@@ -294,7 +323,7 @@ def _build(sources: List[BuildSource],
     data_dir = default_data_dir(bin_dir)
     fscache = fscache or FileSystemCache()
 
-    search_paths = compute_search_paths(sources, options, data_dir, alt_lib_path)
+    search_paths = compute_search_paths(sources, options, data_dir, fscache, alt_lib_path)
 
     reports = Reports(data_dir, options.report_dirs)
     source_set = BuildSourceSet(sources)

mypy/checkstrformat.py

@@ -2,7 +2,7 @@
 
 import re
 
-from typing import cast, List, Tuple, Dict, Callable, Union, Optional
+from typing import cast, List, Tuple, Dict, Callable, Union, Optional, Pattern
 
 from mypy.types import (
     Type, AnyType, TupleType, Instance, UnionType, TypeOfAny
@@ -21,6 +21,20 @@
 Checkers = Tuple[Callable[[Expression], None], Callable[[Type], None]]
 
 
+def compile_format_re() -> Pattern[str]:
+    key_re = r'(\(([^()]*)\))?'  # (optional) parenthesised sequence of characters.
+    flags_re = r'([#0\-+ ]*)'  # (optional) sequence of flags.
+    width_re = r'(\*|[1-9][0-9]*)?'  # (optional) minimum field width (* or numbers).
+    precision_re = r'(?:\.(\*|[0-9]+)?)?'  # (optional) . followed by * of numbers.
+    length_mod_re = r'[hlL]?'  # (optional) length modifier (unused).
+    type_re = r'(.)?'  # conversion type.
+    format_re = '%' + key_re + flags_re + width_re + precision_re + length_mod_re + type_re
+    return re.compile(format_re)
+
+
+FORMAT_RE = compile_format_re()
+
+
 class ConversionSpecifier:
     def __init__(self, key: str, flags: str, width: str, precision: str, type: str) -> None:
         self.key = key
@@ -90,16 +104,8 @@ def check_str_interpolation(self,
             assert False
 
     def parse_conversion_specifiers(self, format: str) -> List[ConversionSpecifier]:
-        key_regex = r'(\(([^()]*)\))?'  # (optional) parenthesised sequence of characters
-        flags_regex = r'([#0\-+ ]*)'  # (optional) sequence of flags
-        width_regex = r'(\*|[1-9][0-9]*)?'  # (optional) minimum field width (* or numbers)
-        precision_regex = r'(?:\.(\*|[0-9]+)?)?'  # (optional) . followed by * of numbers
-        length_mod_regex = r'[hlL]?'  # (optional) length modifier (unused)
-        type_regex = r'(.)?'  # conversion type
-        regex = ('%' + key_regex + flags_regex + width_regex +
-                 precision_regex + length_mod_regex + type_regex)
         specifiers = []  # type: List[ConversionSpecifier]
-        for parens_key, key, flags, width, precision, type in re.findall(regex, format):
+        for parens_key, key, flags, width, precision, type in FORMAT_RE.findall(format):
             if parens_key == '':
                 key = None
             specifiers.append(ConversionSpecifier(key, flags, width, precision, type))

mypy/constraints.py

@@ -312,6 +312,8 @@ def visit_instance(self, template: Instance) -> List[Constraint]:
             actual = actual.as_anonymous().fallback
         if isinstance(actual, Instance):
             instance = actual
+            erased = erase_typevars(template)
+            assert isinstance(erased, Instance)
             # We always try nominal inference if possible,
             # it is much faster than the structural one.
             if (self.direction == SUBTYPE_OF and
@@ -343,8 +345,11 @@ def visit_instance(self, template: Instance) -> List[Constraint]:
                     # This is a conservative way break the inference cycles.
                     # It never produces any "false" constraints but gives up soon
                     # on purely structural inference cycles, see #3829.
+                    # Note that we use is_protocol_implementation instead of is_subtype
+                    # because some type may be considered a subtype of a protocol
+                    # due to _promote, but still not implement the protocol.
                     not any(is_same_type(template, t) for t in template.type.inferring) and
-                    mypy.subtypes.is_subtype(instance, erase_typevars(template))):
+                    mypy.subtypes.is_protocol_implementation(instance, erased)):
                 template.type.inferring.append(template)
                 self.infer_constraints_from_protocol_members(res, instance, template,
                                                              original_actual, template)
@@ -353,7 +358,7 @@ def visit_instance(self, template: Instance) -> List[Constraint]:
             elif (instance.type.is_protocol and self.direction == SUBTYPE_OF and
                   # We avoid infinite recursion for structural subtypes also here.
                   not any(is_same_type(instance, i) for i in instance.type.inferring) and
-                  mypy.subtypes.is_subtype(erase_typevars(template), instance)):
+                  mypy.subtypes.is_protocol_implementation(erased, instance)):
                 instance.type.inferring.append(instance)
                 self.infer_constraints_from_protocol_members(res, instance, template,
                                                              template, instance)

mypy/dmypy_server.py

@@ -327,7 +327,7 @@ def fine_grained_increment(self, sources: List[mypy.build.BuildSource]) -> Dict[
         self.update_sources(sources)
         changed, removed = self.find_changed(sources)
         manager.search_paths = mypy.build.compute_search_paths(
-            sources, manager.options, manager.data_dir)
+            sources, manager.options, manager.data_dir, mypy.build.FileSystemCache())
         t1 = time.time()
         messages = self.fine_grained_manager.update(changed, removed)
         t2 = time.time()

mypy/fixup.py

@@ -279,11 +279,11 @@ def lookup_qualified_stnode(modules: Dict[str, MypyFile], name: str,
             return stnode
         node = stnode.node
         # In fine-grained mode, could be a cross-reference to a deleted module
-        if node is None:
+        # or a Var made up for a missing module.
+        if not isinstance(node, TypeInfo):
             if not quick_and_dirty:
                 assert node, "Cannot find %s" % (name,)
             return None
-        assert isinstance(node, TypeInfo)
         names = node.names
 
 

mypy/semanal.py

@@ -855,6 +855,7 @@ def calculate_abstract_status(self, typ: TypeInfo) -> None:
         """
         concrete = set()  # type: Set[str]
         abstract = []  # type: List[str]
+        abstract_in_this_class = []  # type: List[str]
         for base in typ.mro:
             for name, symnode in base.names.items():
                 node = symnode.node
@@ -871,12 +872,30 @@ def calculate_abstract_status(self, typ: TypeInfo) -> None:
                     if fdef.is_abstract and name not in concrete:
                         typ.is_abstract = True
                         abstract.append(name)
+                        if base is typ:
+                            abstract_in_this_class.append(name)
                 elif isinstance(node, Var):
                     if node.is_abstract_var and name not in concrete:
                         typ.is_abstract = True
                         abstract.append(name)
+                        if base is typ:
+                            abstract_in_this_class.append(name)
                 concrete.add(name)
+        # In stubs, abstract classes need to be explicitly marked because it is too
+        # easy to accidentally leave a concrete class abstract by forgetting to
+        # implement some methods.
         typ.abstract_attributes = sorted(abstract)
+        if not self.is_stub_file:
+            return
+        if (typ.declared_metaclass and typ.declared_metaclass.type.fullname() == 'abc.ABCMeta'):
+            return
+        if typ.is_protocol:
+            return
+        if abstract and not abstract_in_this_class:
+            attrs = ", ".join('"{}"'.format(attr) for attr in sorted(abstract))
+            self.fail("Class {} has abstract attributes {}".format(typ.fullname(), attrs), typ)
+            self.note("If it is meant to be abstract, add 'abc.ABCMeta' as an explicit metaclass",
+                      typ)
 
     def setup_type_promotion(self, defn: ClassDef) -> None:
         """Setup extra, ad-hoc subtyping relationships between classes (promotion).
@@ -1386,19 +1405,14 @@ def visit_import_from(self, imp: ImportFrom) -> None:
             # If it is still not resolved, check for a module level __getattr__
             if (module and not node and (module.is_stub or self.options.python_version >= (3, 7))
                     and '__getattr__' in module.names):
-                getattr_defn = module.names['__getattr__']
-                if isinstance(getattr_defn.node, (FuncDef, Var)):
-                    if isinstance(getattr_defn.node.type, CallableType):
-                        typ = getattr_defn.node.type.ret_type
-                    else:
-                        typ = AnyType(TypeOfAny.from_error)
-                    if as_id:
-                        name = as_id
-                    else:
-                        name = id
-                    ast_node = Var(name, type=typ)
-                    symbol = SymbolTableNode(GDEF, ast_node)
-                    self.add_symbol(name, symbol, imp)
+                name = as_id if as_id else id
+                if self.type:
+                    fullname = self.type.fullname() + "." + name
+                else:
+                    fullname = self.qualified_name(name)
+                gvar = self.create_getattr_var(module.names['__getattr__'], name, fullname)
+                if gvar:
+                    self.add_symbol(name, gvar, imp)
                     continue
             if node and node.kind != UNBOUND_IMPORTED and not node.module_hidden:
                 if not node:
@@ -3064,7 +3078,15 @@ def lookup_qualified(self, name: str, ctx: Context,
                         n = names.get(parts[i], None)
                         if n and isinstance(n.node, ImportedName):
                             n = self.dereference_module_cross_ref(n)
+                        elif '__getattr__' in names:
+                            gvar = self.create_getattr_var(names['__getattr__'],
+                                                           parts[i], parts[i])
+                            if gvar:
+                                names[name] = gvar
+                                n = gvar
                     # TODO: What if node is Var or FuncDef?
+                    # Currently, missing these cases results in controversial behavior, when
+                    # lookup_qualified(x.y.z) returns Var(x).
                     if not n:
                         if not suppress_errors:
                             self.name_not_defined(name, ctx)
@@ -3077,6 +3099,22 @@ def lookup_qualified(self, name: str, ctx: Context,
                 return n
             return None
 
+    def create_getattr_var(self, getattr_defn: SymbolTableNode,
+                           name: str, fullname: str) -> Optional[SymbolTableNode]:
+        """Create a dummy global symbol using __getattr__ return type.
+
+        If not possible, return None.
+        """
+        if isinstance(getattr_defn.node, (FuncDef, Var)):
+            if isinstance(getattr_defn.node.type, CallableType):
+                typ = getattr_defn.node.type.ret_type
+            else:
+                typ = AnyType(TypeOfAny.from_error)
+            v = Var(name, type=typ)
+            v._fullname = fullname
+            return SymbolTableNode(GDEF, v)
+        return None
+
     def rebind_symbol_table_node(self, n: SymbolTableNode) -> Optional[SymbolTableNode]:
         """If node refers to old version of module, return reference to new version.
 

mypy/stubutil.py

@@ -1,5 +1,6 @@
 import re
 import sys
+import os
 
 from typing import Optional, Tuple, Sequence, MutableSequence, List, MutableMapping, IO
 from types import ModuleType
@@ -87,7 +88,8 @@ def find_unique_signatures(sigs: Sequence[Sig]) -> List[Sig]:
 
 
 def is_c_module(module: ModuleType) -> bool:
-    return '__file__' not in module.__dict__ or module.__dict__['__file__'].endswith('.so')
+    return ('__file__' not in module.__dict__ or
+            os.path.splitext(module.__dict__['__file__'])[-1] in ['.so', '.pyd'])
 
 
 def write_header(file: IO[str], module_name: Optional[str] = None,

mypy/subtypes.py

@@ -524,7 +524,7 @@ def get_member_flags(name: str, info: TypeInfo) -> Set[int]:
         if v.var.is_staticmethod or v.var.is_classmethod:
             return {IS_CLASS_OR_STATIC}
     # just a variable
-    if isinstance(v, Var):
+    if isinstance(v, Var) and not v.is_property:
         flags = {IS_SETTABLE}
         if v.is_classvar:
             flags.add(IS_CLASSVAR)