Handle types.ModuleType (#3107)

* Allowed use of types.ModuleType

* Allow module name override for stub files

* Remove superfluous class module

* Hard code module renaming

* Address CR

* Updated typeshed

* Fix fine-grained tests

* Fix whitespace

Author: pkch

mypy/checkexpr.py

@@ -151,7 +151,7 @@ def analyze_ref_expr(self, e: RefExpr, lvalue: bool = False) -> Type:
             result = type_object_type(node, self.named_type)
         elif isinstance(node, MypyFile):
             # Reference to a module object.
-            result = self.named_type('builtins.module')
+            result = self.named_type('types.ModuleType')
         elif isinstance(node, Decorator):
             result = self.analyze_var_ref(node.var, e)
         else:

mypy/checkmember.py

@@ -415,7 +415,7 @@ def analyze_class_attribute_access(itype: Instance,
 
     if isinstance(node.node, MypyFile):
         # Reference to a module object.
-        return builtin_type('builtins.module')
+        return builtin_type('types.ModuleType')
 
     if is_decorated:
         # TODO: Return type of decorated function. This is quick hack to work around #998.

mypy/semanal.py

@@ -113,6 +113,13 @@
     'typing.typevar': 'typing.TypeVar',
 }
 
+# Rename objects placed in _importlib_modulespec due to circular imports
+module_rename_map = {
+    '_importlib_modulespec.ModuleType': 'types.ModuleType',
+    '_importlib_modulespec.ModuleSpec': 'importlib.machinery.ModuleSpec',
+    '_importlib_modulespec.Loader': 'importlib.abc.Loader'
+}
+
 # Hard coded type promotions (shared between all Python versions).
 # These add extra ad-hoc edges to the subtyping relation. For example,
 # int is considered a subtype of float, even though there is no
@@ -3445,6 +3452,8 @@ def visit_file(self, file: MypyFile, fnam: str, mod_id: str, options: Options) -
 
         for d in defs:
             d.accept(self)
+            if isinstance(d, ClassDef):
+                d.info._fullname = module_rename_map.get(d.info._fullname, d.info._fullname)
 
         # Add implicit definition of literals/keywords to builtins, as we
         # cannot define a variable with them explicitly.

mypy/server/deps.py

@@ -207,10 +207,12 @@ def visit_tuple_type(self, typ: TupleType) -> List[str]:
         raise NotImplementedError
 
     def visit_type_type(self, typ: TypeType) -> List[str]:
-        raise NotImplementedError
+        # TODO: replace with actual implementation
+        return []
 
     def visit_type_var(self, typ: TypeVarType) -> List[str]:
-        raise NotImplementedError
+        # TODO: replace with actual implementation
+        return []
 
     def visit_typeddict_type(self, typ: TypedDictType) -> List[str]:
         raise NotImplementedError

test-data/unit/check-ignore.test

@@ -46,7 +46,7 @@ import b # type: ignore
 reveal_type(a.foo) # E: Revealed type is 'Any'
 reveal_type(b.foo) # E: Revealed type is 'builtins.int'
 a.bar()
-b.bar() # E: "module" has no attribute "bar"
+b.bar() # E: "ModuleType" has no attribute "bar"
 
 [file b.py]
 foo = 3
@@ -76,7 +76,7 @@ class B(A):
 import m
 m.x = object # type: ignore
 m.f() # type: ignore
-m.y # E: "module" has no attribute "y"
+m.y # E: "ModuleType" has no attribute "y"
 [file m.py]
 [builtins fixtures/module.pyi]
 

test-data/unit/check-incremental.test

@@ -300,7 +300,7 @@ const = 3
 [stale mod3]
 [builtins fixtures/module.pyi]
 [out2]
-tmp/mod1.py:3: error: "module" has no attribute "mod4"
+tmp/mod1.py:3: error: "ModuleType" has no attribute "mod4"
 
 [case testIncrementalLongBrokenCascade]
 import mod1
@@ -335,7 +335,7 @@ const = 3
 [stale mod6]
 [builtins fixtures/module.pyi]
 [out2]
-tmp/mod1.py:3: error: "module" has no attribute "mod7"
+tmp/mod1.py:3: error: "ModuleType" has no attribute "mod7"
 
 [case testIncrementalNestedBrokenCascade]
 import mod1
@@ -361,7 +361,7 @@ const = 3
 [stale mod2.mod3]
 [builtins fixtures/module.pyi]
 [out2]
-tmp/mod1.py:3: error: "module" has no attribute "mod4"
+tmp/mod1.py:3: error: "ModuleType" has no attribute "mod4"
 
 [case testIncrementalNestedBrokenCascadeWithType1]
 import mod1, mod2.mod3.mod5

test-data/unit/check-modules.test

@@ -153,17 +153,18 @@ def f(c:str) -> None: pass
 [case testInvalidOperationsOnModules]
 import m
 import typing
+
 class A: pass
-m()      # E: "module" not callable
-a = m # type: A  # E: Incompatible types in assignment (expression has type "module", variable has type "A")
-m + None # E: Unsupported left operand type for + ("module")
+m()      # E: "ModuleType" not callable
+a = m # type: A  # E: Incompatible types in assignment (expression has type "ModuleType", variable has type "A")
+m + None # E: Unsupported left operand type for + ("ModuleType")
 [file m.py]
 [builtins fixtures/module.pyi]
 
 [case testNameDefinedInDifferentModule]
 import m, n
 import typing
-m.x # E: "module" has no attribute "x"
+m.x # E: "ModuleType" has no attribute "x"
 [file m.py]
 y = object()
 [file n.py]
@@ -329,7 +330,7 @@ import nonexistent
 [out]
 tmp/x.py:1: error: Cannot find module named 'nonexistent'
 tmp/x.py:1: note: (Perhaps setting MYPYPATH or using the "--ignore-missing-imports" flag would help)
-main:3: error: "module" has no attribute "z"
+main:3: error: "ModuleType" has no attribute "z"
 
 [case testUnknownModuleImportedWithinFunction]
 def f():
@@ -647,7 +648,7 @@ def f(x: str) -> None: pass
 if object():
     import m
 else:
-    m = 1 # E: Incompatible types in assignment (expression has type "int", variable has type "module")
+    m = 1 # E: Incompatible types in assignment (expression has type "int", variable has type "ModuleType")
 [file m.py]
 [builtins fixtures/module.pyi]
 [out]
@@ -751,7 +752,7 @@ value = 3.2
 [case testSubmoduleImportFromDoesNotAddParents]
 from a import b
 reveal_type(b.value)  # E: Revealed type is 'builtins.str'
-b.c.value  # E: "module" has no attribute "c"
+b.c.value  # E: "ModuleType" has no attribute "c"
 a.value  # E: Name 'a' is not defined
 
 [file a/__init__.py]
@@ -852,7 +853,7 @@ bar = parent.unrelated.ShouldNotLoad()
 [builtins fixtures/module.pyi]
 [out]
 tmp/parent/child.py:8: error: Revealed type is 'parent.common.SomeClass'
-tmp/parent/child.py:9: error: "module" has no attribute "unrelated"
+tmp/parent/child.py:9: error: "ModuleType" has no attribute "unrelated"
 
 [case testSubmoduleMixingImportFromAndImport2]
 import parent.child
@@ -1406,3 +1407,12 @@ reveal_type(cb) # E: Revealed type is 'def (*Any, **Any) -> Any'
 from typing import Callable, Any
 AnyCallable = Callable[..., Any]
 [out]
+
+[case testRevealType]
+import types
+def f() -> types.ModuleType:
+    return types
+reveal_type(f())  # E: Revealed type is 'types.ModuleType'
+reveal_type(types)  # E: Revealed type is 'types.ModuleType'
+
+[builtins fixtures/module.pyi]

test-data/unit/cmdline.test

@@ -365,8 +365,8 @@ x += ''  # Error reported here
 a.py:2: error: Unsupported operand types for + ("int" and "str")
 main.py:3: error: Unsupported operand types for + ("int" and "str")
 main.py:6: error: Unsupported operand types for + ("int" and "str")
-main.py:7: error: "module" has no attribute "y"
-main.py:8: error: Unsupported operand types for + ("module" and "int")
+main.py:7: error: "ModuleType" has no attribute "y"
+main.py:8: error: Unsupported operand types for + ("ModuleType" and "int")
 
 [case testConfigFollowImportsSilent]
 # cmd: mypy main.py
@@ -386,8 +386,8 @@ x += ''  # No error reported
 [out]
 main.py:2: error: Unsupported operand types for + ("int" and "str")
 main.py:4: error: Unsupported operand types for + ("int" and "str")
-main.py:5: error: "module" has no attribute "y"
-main.py:6: error: Unsupported operand types for + ("module" and "int")
+main.py:5: error: "ModuleType" has no attribute "y"
+main.py:6: error: Unsupported operand types for + ("ModuleType" and "int")
 
 [case testConfigFollowImportsSkip]
 # cmd: mypy main.py

test-data/unit/fine-grained.test

@@ -68,7 +68,7 @@ def g(x: str) -> None: pass
 [builtins fixtures/fine_grained.pyi]
 [out]
 ==
-main:3: error: "module" has no attribute "f"
+main:3: error: "ModuleType" has no attribute "f"
 
 [case testTopLevelMissingModuleAttribute]
 import m
@@ -81,7 +81,7 @@ def g(x: int) -> None: pass
 [builtins fixtures/fine_grained.pyi]
 [out]
 ==
-main:2: error: "module" has no attribute "f"
+main:2: error: "ModuleType" has no attribute "f"
 
 [case testClassChangedIntoFunction]
 import m
@@ -241,7 +241,7 @@ class A: pass
 [builtins fixtures/fine_grained.pyi]
 [out]
 ==
-main:3: error: "module" has no attribute "A"
+main:3: error: "ModuleType" has no attribute "A"
 ==
 
 [case testContinueToReportTypeCheckError]
@@ -281,10 +281,10 @@ class A: pass
 [builtins fixtures/fine_grained.pyi]
 [out]
 ==
-main:3: error: "module" has no attribute "A"
-main:5: error: "module" has no attribute "B"
+main:3: error: "ModuleType" has no attribute "A"
+main:5: error: "ModuleType" has no attribute "B"
 ==
-main:5: error: "module" has no attribute "B"
+main:5: error: "ModuleType" has no attribute "B"
 
 [case testContinueToReportErrorAtTopLevel]
 import n
@@ -348,9 +348,9 @@ def g() -> None: pass
 [builtins fixtures/fine_grained.pyi]
 [out]
 main:3: error: Too few arguments for "f"
-main:5: error: "module" has no attribute "g"
+main:5: error: "ModuleType" has no attribute "g"
 ==
-main:5: error: "module" has no attribute "g"
+main:5: error: "ModuleType" has no attribute "g"
 ==
 
 [case testKeepReportingErrorIfNoChanges]
@@ -361,9 +361,9 @@ def h() -> None:
 [file m.py.2]
 [builtins fixtures/fine_grained.pyi]
 [out]
-main:3: error: "module" has no attribute "g"
+main:3: error: "ModuleType" has no attribute "g"
 ==
-main:3: error: "module" has no attribute "g"
+main:3: error: "ModuleType" has no attribute "g"
 
 [case testFixErrorAndReintroduce]
 import m
@@ -375,10 +375,10 @@ def g() -> None: pass
 [file m.py.3]
 [builtins fixtures/fine_grained.pyi]
 [out]
-main:3: error: "module" has no attribute "g"
+main:3: error: "ModuleType" has no attribute "g"
 ==
 ==
-main:3: error: "module" has no attribute "g"
+main:3: error: "ModuleType" has no attribute "g"
 
 [case testAddBaseClassMethodCausingInvalidOverride]
 import m

test-data/unit/fixtures/args.pyi

@@ -26,4 +26,3 @@ class int:
 class str: pass
 class bool: pass
 class function: pass
-class module: pass

test-data/unit/fixtures/async_await.pyi

@@ -1,16 +1,20 @@
 import typing
+
+T = typing.TypeVar('T')
+class list(typing.Generic[T], typing.Sequence[T]): pass
+
 class object:
     def __init__(self): pass
 class type: pass
 class function: pass
 class int: pass
 class str: pass
 class dict: pass
-class list: pass
 class set: pass
 class tuple: pass
 class BaseException: pass
 class StopIteration(BaseException): pass
 class StopAsyncIteration(BaseException): pass
 def iter(obj: typing.Any) -> typing.Any: pass
 def next(obj: typing.Any) -> typing.Any: pass
+class ellipsis: ...

test-data/unit/fixtures/fine_grained.pyi

@@ -3,6 +3,8 @@
 # TODO: Migrate to regular stubs once fine-grained incremental is robust
 #       enough to handle them.
 
+import types
+
 class Any: pass
 
 class object:
@@ -21,4 +23,4 @@ class bytes: pass
 class tuple: pass
 class function: pass
 class ellipsis: pass
-class module: pass
+class list: pass

test-data/unit/fixtures/module.pyi

@@ -1,18 +1,19 @@
-from typing import Any, Dict, Generic, TypeVar
+from typing import Any, Dict, Generic, TypeVar, Sequence
+from types import ModuleType
 
 T = TypeVar('T')
 S = TypeVar('S')
 
+class list(Generic[T], Sequence[T]): pass
+
 class object:
     def __init__(self) -> None: pass
-class module:
-    __name__ = ...  # type: str
-    __file__ = ...  # type: str
-    __dict__ = ...  # type: Dict[str, Any]
 class type: pass
 class function: pass
 class int: pass
 class str: pass
 class bool: pass
 class tuple: pass
 class dict(Generic[T, S]): pass
+class ellipsis: pass
+

test-data/unit/fixtures/module_all.pyi

@@ -1,15 +1,18 @@
 from typing import Generic, Sequence, TypeVar
+from types import ModuleType
+
 _T = TypeVar('_T')
 
 class object:
     def __init__(self) -> None: pass
-class module: pass
 class type: pass
 class function: pass
 class int: pass
 class str: pass
+class bool: pass
 class list(Generic[_T], Sequence[_T]):
     def append(self, x: _T): pass
     def extend(self, x: Sequence[_T]): pass
     def __add__(self, rhs: Sequence[_T]) -> list[_T]: pass
 class tuple: pass
+class ellipsis: pass

test-data/unit/fixtures/module_all_python2.pyi

@@ -3,7 +3,6 @@ _T = TypeVar('_T')
 
 class object:
     def __init__(self) -> None: pass
-class module: pass
 class type: pass
 class function: pass
 class int: pass

test-data/unit/fixtures/ops.pyi

@@ -54,5 +54,3 @@ class float: pass
 class BaseException: pass
 
 def __print(a1=None, a2=None, a3=None, a4=None): pass
-
-class module: pass

test-data/unit/lib-stub/types.pyi

@@ -1,4 +1,9 @@
-from typing import TypeVar
+from typing import TypeVar, Optional, List, Any, Generic, Sequence
 T = TypeVar('T')
+
 def coroutine(func: T) -> T:
     return func
+
+class bool: ...
+
+class ModuleType: ...

test-data/unit/pythoneval.test

@@ -1169,8 +1169,8 @@ collections.Deque()
 typing.deque()
 
 [out]
-_testDequeWrongCase.py:4: error: "module" has no attribute "Deque"
-_testDequeWrongCase.py:5: error: "module" has no attribute "deque"
+_testDequeWrongCase.py:4: error: "ModuleType" has no attribute "Deque"
+_testDequeWrongCase.py:5: error: "ModuleType" has no attribute "deque"
 
 [case testDictUpdateInference]
 from typing import Dict, Optional