From 2fa540a7875620fb32e8212abf05bdde2b6bbea6 Mon Sep 17 00:00:00 2001 From: Ran Benita Date: Sat, 3 Aug 2019 14:50:15 +0300 Subject: [PATCH 1/5] scripts/dumpmodule: drop Python 2 compatibility --- scripts/dumpmodule.py | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/scripts/dumpmodule.py b/scripts/dumpmodule.py index ac8470ece763..11107a2e94da 100644 --- a/scripts/dumpmodule.py +++ b/scripts/dumpmodule.py @@ -1,25 +1,12 @@ """Dump the runtime structure of a module as JSON. This is used for testing stubs. - -This needs to run in Python 2.7 and 3.x. """ - -from __future__ import print_function - import importlib import json import sys import types -from typing import Text - - -if sys.version_info >= (3, 0): - import inspect - long = int -else: - import inspect2 as inspect - +import inspect def dump_module(id): @@ -75,7 +62,7 @@ def dump_value(value, depth=0): def dump_simple(value): - if type(value) in (int, bool, float, str, bytes, Text, long, list, set, dict, tuple): + if type(value) in (int, bool, float, str, bytes, list, set, dict, tuple): return {'type': type(value).__name__} if value is None: return {'type': 'None'} From 9706e75021579706e20720248b3945d74522eafe Mon Sep 17 00:00:00 2001 From: Ran Benita Date: Sat, 3 Aug 2019 15:06:32 +0300 Subject: [PATCH 2/5] scripts/dumpmodule: add type annotations --- scripts/dumpmodule.py | 39 +++++++++++++++++++++++---------------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/scripts/dumpmodule.py b/scripts/dumpmodule.py index 11107a2e94da..f0313de6d747 100644 --- a/scripts/dumpmodule.py +++ b/scripts/dumpmodule.py @@ -3,20 +3,26 @@ This is used for testing stubs. """ import importlib +import inspect import json import sys import types -import inspect +from types import FunctionType +from typing import Optional, Dict, Any, Set, Callable +from typing_extensions import Final + +DumpNode = Dict[str, Any] -def dump_module(id): + +def dump_module(id: str) -> None: m = importlib.import_module(id) data = module_to_json(m) print(json.dumps(data, ensure_ascii=True, indent=4, sort_keys=True)) -def module_to_json(m): - result = {} +def module_to_json(m: object) -> Dict[str, DumpNode]: + result = {} # type: Dict[str, DumpNode] for name, value in m.__dict__.items(): # Filter out some useless attributes. @@ -33,7 +39,7 @@ def module_to_json(m): result[name] = dump_value(value) try: - _, line = inspect.getsourcelines(getattr(m, name)) + line = inspect.getsourcelines(getattr(m, name))[1] # type: Optional[int] except (TypeError, OSError): line = None @@ -42,12 +48,13 @@ def module_to_json(m): return result -def dump_value(value, depth=0): +def dump_value(value: object, depth: int = 0) -> DumpNode: if depth > 10: - return 'max_recursion_depth_exceeded' + # TODO: Callers don't handle this case. + return 'max_recursion_depth_exceeded' # type: ignore if isinstance(value, type): return dump_class(value, depth + 1) - if inspect.isfunction(value): + if isinstance(value, FunctionType): return dump_function(value) if callable(value): return {'type': 'callable'} # TODO more information @@ -61,7 +68,7 @@ def dump_value(value, depth=0): return dump_simple(value) -def dump_simple(value): +def dump_simple(value: object) -> DumpNode: if type(value) in (int, bool, float, str, bytes, list, set, dict, tuple): return {'type': type(value).__name__} if value is None: @@ -71,7 +78,7 @@ def dump_simple(value): return {'type': 'unknown'} -def dump_class(value, depth): +def dump_class(value: type, depth: int) -> DumpNode: return { 'type': 'class', 'attributes': dump_attrs(value, depth), @@ -86,13 +93,13 @@ def dump_class(value, depth): '__bool__', '__contains__', '__iter__', -] +] # type: Final # Change to return a dict -def dump_attrs(d, depth): +def dump_attrs(d: type, depth: int) -> DumpNode: result = {} - seen = set() + seen = set() # type: Set[str] try: mro = d.mro() except TypeError: @@ -115,10 +122,10 @@ def dump_attrs(d, depth): inspect.Parameter.VAR_POSITIONAL: 'VAR_POS', inspect.Parameter.KEYWORD_ONLY: 'KW_ONLY', inspect.Parameter.VAR_KEYWORD: 'VAR_KW', -} +} # type: Final -def param_kind(p): +def param_kind(p: inspect.Parameter) -> str: s = kind_map[p.kind] if p.default != inspect.Parameter.empty: assert s in ('POS_ONLY', 'POS_OR_KW', 'KW_ONLY') @@ -126,7 +133,7 @@ def param_kind(p): return s -def dump_function(value): +def dump_function(value: FunctionType) -> DumpNode: try: sig = inspect.signature(value) except ValueError: From 43dc86e919697c7d0b96d63bb00a9e7eea7da53c Mon Sep 17 00:00:00 2001 From: Ran Benita Date: Sat, 27 Jul 2019 13:55:15 +0300 Subject: [PATCH 3/5] scripts/stubtest: make it run again This script used some internal APIs which have been removed. --- scripts/stubtest.py | 35 ++++++++++++++++++++--------------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/scripts/stubtest.py b/scripts/stubtest.py index 0ec72cef44f3..069f049d062a 100644 --- a/scripts/stubtest.py +++ b/scripts/stubtest.py @@ -6,11 +6,12 @@ import importlib import sys -from typing import Dict, Any +from typing import Dict, Any, List, Iterator from collections import defaultdict, namedtuple from mypy import build -from mypy.build import default_data_dir, default_lib_path, find_modules_recursive +from mypy.build import default_data_dir +from mypy.modulefinder import compute_search_paths, FindModuleCache from mypy.errors import CompileError from mypy import nodes from mypy.options import Options @@ -54,9 +55,11 @@ 'module_type')) -def test_stub(name: str): +def test_stub(options: Options, + find_module_cache: FindModuleCache, + name: str) -> Iterator[Error]: stubs = { - mod: stub for mod, stub in build_stubs(name).items() + mod: stub for mod, stub in build_stubs(options, find_module_cache, name).items() if (mod == name or mod.startswith(name + '.')) and mod not in skip } @@ -152,7 +155,7 @@ def verify_typevarexpr(node, module_node): @verify.register(nodes.Decorator) -def verify_decorator(node, module_noode): +def verify_decorator(node, module_node): if False: yield None @@ -162,14 +165,10 @@ def dump_module(name: str) -> Dict[str, Any]: return {'type': 'file', 'names': dumpmodule.module_to_json(mod)} -def build_stubs(mod): - data_dir = default_data_dir(None) - options = Options() - options.python_version = (3, 6) - lib_path = default_lib_path(data_dir, - options.python_version, - custom_typeshed_dir=None) - sources = find_modules_recursive(mod, lib_path) +def build_stubs(options: Options, + find_module_cache: FindModuleCache, + mod: str) -> Dict[str, nodes.MypyFile]: + sources = find_module_cache.find_modules_recursive(mod) try: res = build.build(sources=sources, options=options) @@ -184,15 +183,21 @@ def build_stubs(mod): return res.files -def main(args): +def main(args: List[str]) -> Iterator[Error]: if len(args) == 1: print('must provide at least one module to test') sys.exit(1) else: modules = args[1:] + options = Options() + options.python_version = (3, 6) + data_dir = default_data_dir() + search_path = compute_search_paths([], options, data_dir) + find_module_cache = FindModuleCache(search_path) + for module in modules: - for error in test_stub(module): + for error in test_stub(options, find_module_cache, module): yield error From e4c2ca7e912e8da878fb0e3e6bbc2384bae88b4d Mon Sep 17 00:00:00 2001 From: Ran Benita Date: Sat, 3 Aug 2019 13:55:41 +0300 Subject: [PATCH 4/5] scripts/stubtest: add more complete type annotations --- scripts/stubtest.py | 87 ++++++++++++++++++++++++++++----------------- 1 file changed, 54 insertions(+), 33 deletions(-) diff --git a/scripts/stubtest.py b/scripts/stubtest.py index 069f049d062a..5bbadb2f4cbe 100644 --- a/scripts/stubtest.py +++ b/scripts/stubtest.py @@ -6,8 +6,10 @@ import importlib import sys -from typing import Dict, Any, List, Iterator -from collections import defaultdict, namedtuple +from typing import Dict, Any, List, Iterator, NamedTuple, Optional, Mapping, Tuple +from typing_extensions import Type, Final +from collections import defaultdict +from functools import singledispatch from mypy import build from mypy.build import default_data_dir @@ -16,9 +18,8 @@ from mypy import nodes from mypy.options import Options -import dumpmodule +from dumpmodule import module_to_json, DumpNode -from functools import singledispatch # TODO: email.contentmanager has a symbol table with a None node. # This seems like it should not be. @@ -34,7 +35,8 @@ 'unittest.mock', # mock.call infinite loops on inspect.getsourcelines # https://bugs.python.org/issue25532 # TODO: can we filter only call? -} +} # type: Final + messages = { 'not_in_runtime': ('{error.stub_type} "{error.name}" defined at line ' @@ -44,15 +46,24 @@ 'no_stubs': 'could not find typeshed {error.name}', 'inconsistent': ('"{error.name}" is {error.stub_type} in stub but' ' {error.module_type} at runtime'), -} - -Error = namedtuple('Error', ( - 'module', - 'name', - 'error_type', - 'line', - 'stub_type', - 'module_type')) +} # type: Final + +Error = NamedTuple('Error', ( + ('module', str), + ('name', str), + ('error_type', str), + ('line', Optional[int]), + ('stub_type', Optional[Type[nodes.Node]]), + ('module_type', Optional[str]), +)) + +ErrorParts = Tuple[ + List[str], + str, + Optional[int], + Optional[Type[nodes.Node]], + Optional[str], +] def test_stub(options: Options, @@ -66,24 +77,26 @@ def test_stub(options: Options, for mod, stub in stubs.items(): instance = dump_module(mod) - for identifiers, *error in verify(stub, instance): - yield Error(mod, '.'.join(identifiers), *error) + for identifiers, error_type, line, stub_type, module_type in verify(stub, instance): + yield Error(mod, '.'.join(identifiers), error_type, line, stub_type, module_type) @singledispatch -def verify(node, module_node): +def verify(node: nodes.Node, + module_node: Optional[DumpNode]) -> Iterator[ErrorParts]: raise TypeError('unknown mypy node ' + str(node)) @verify.register(nodes.MypyFile) -def verify_mypyfile(stub, instance): +def verify_mypyfile(stub: nodes.MypyFile, + instance: Optional[DumpNode]) -> Iterator[ErrorParts]: if instance is None: yield [], 'not_in_runtime', stub.line, type(stub), None elif instance['type'] != 'file': yield [], 'inconsistent', stub.line, type(stub), instance['type'] else: - stub_children = defaultdict(lambda: None, stub.names) + stub_children = defaultdict(lambda: None, stub.names) # type: Mapping[str, Optional[nodes.SymbolTableNode]] instance_children = defaultdict(lambda: None, instance['names']) # TODO: I would rather not filter public children here. @@ -93,16 +106,18 @@ def verify_mypyfile(stub, instance): name: (stub_children[name], instance_children[name]) for name in set(stub_children) | set(instance_children) if not name.startswith('_') - and (stub_children[name] is None or stub_children[name].module_public) + and (stub_children[name] is None or stub_children[name].module_public) # type: ignore } for node, (stub_child, instance_child) in public_nodes.items(): stub_child = getattr(stub_child, 'node', None) - for identifiers, *error in verify(stub_child, instance_child): - yield ([node] + identifiers, *error) + for identifiers, error_type, line, stub_type, module_type in verify(stub_child, instance_child): + yield ([node] + identifiers, error_type, line, stub_type, module_type) + @verify.register(nodes.TypeInfo) -def verify_typeinfo(stub, instance): +def verify_typeinfo(stub: nodes.TypeInfo, + instance: Optional[DumpNode]) -> Iterator[ErrorParts]: if not instance: yield [], 'not_in_runtime', stub.line, type(stub), None elif instance['type'] != 'class': @@ -110,12 +125,13 @@ def verify_typeinfo(stub, instance): else: for attr, attr_node in stub.names.items(): subdump = instance['attributes'].get(attr, None) - for identifiers, *error in verify(attr_node.node, subdump): - yield ([attr] + identifiers, *error) + for identifiers, error_type, line, stub_type, module_type in verify(attr_node.node, subdump): + yield ([attr] + identifiers, error_type, line, stub_type, module_type) @verify.register(nodes.FuncItem) -def verify_funcitem(stub, instance): +def verify_funcitem(stub: nodes.FuncItem, + instance: Optional[DumpNode]) -> Iterator[ErrorParts]: if not instance: yield [], 'not_in_runtime', stub.line, type(stub), None elif 'type' not in instance or instance['type'] not in ('function', 'callable'): @@ -124,7 +140,8 @@ def verify_funcitem(stub, instance): @verify.register(type(None)) -def verify_none(stub, instance): +def verify_none(stub: None, + instance: Optional[DumpNode]) -> Iterator[ErrorParts]: if instance is None: yield [], 'not_in_stub', None, None, None else: @@ -132,7 +149,8 @@ def verify_none(stub, instance): @verify.register(nodes.Var) -def verify_var(node, module_node): +def verify_var(node: nodes.Var, + module_node: Optional[DumpNode]) -> Iterator[ErrorParts]: if False: yield None # Need to check if types are inconsistent. @@ -142,27 +160,30 @@ def verify_var(node, module_node): @verify.register(nodes.OverloadedFuncDef) -def verify_overloadedfuncdef(node, module_node): +def verify_overloadedfuncdef(node: nodes.OverloadedFuncDef, + module_node: Optional[DumpNode]) -> Iterator[ErrorParts]: # Should check types of the union of the overloaded types. if False: yield None @verify.register(nodes.TypeVarExpr) -def verify_typevarexpr(node, module_node): +def verify_typevarexpr(node: nodes.TypeVarExpr, + module_node: Optional[DumpNode]) -> Iterator[ErrorParts]: if False: yield None @verify.register(nodes.Decorator) -def verify_decorator(node, module_node): +def verify_decorator(node: nodes.Decorator, + module_node: Optional[DumpNode]) -> Iterator[ErrorParts]: if False: yield None -def dump_module(name: str) -> Dict[str, Any]: +def dump_module(name: str) -> DumpNode: mod = importlib.import_module(name) - return {'type': 'file', 'names': dumpmodule.module_to_json(mod)} + return {'type': 'file', 'names': module_to_json(mod)} def build_stubs(options: Options, From 5144c6698bf172dc4410cf2d8d422030bf6fcbcb Mon Sep 17 00:00:00 2001 From: Ran Benita Date: Sat, 3 Aug 2019 14:46:04 +0300 Subject: [PATCH 5/5] scripts/stubtest: type-check it in tox -e type --- tox.ini | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 16ddb5fea8a8..d6ae7185fd04 100644 --- a/tox.ini +++ b/tox.ini @@ -49,7 +49,9 @@ commands = flake8 {posargs} [testenv:type] description = type check ourselves basepython = python3.7 -commands = python -m mypy --config-file mypy_self_check.ini -p mypy +commands = + python -m mypy --config-file mypy_self_check.ini -p mypy + python -m mypy --config-file mypy_self_check.ini scripts/stubtest.py [testenv:docs] description = invoke sphinx-build to build the HTML docs