Report precision stats for imports and add new precision report (#7254)

Previously when reporting per-line type checking precision, imports were
treated as empty lines, which seems inconsistent. Now treat an import as
precise if the target module exists, and as imprecise if the target module is
missing from the build.

Also add a new report, `lineprecision`, which is mostly designed to be used
in tests. The output is more compact and cleaner than the existing
XML-based reports.

Make it possible to run report generation tests without using full stubs,
since using full stubs is slow.

This is still not quite perfect. If using module-level `__getattr__`, the import
should probably be treated as imprecise. Also, if a missing name is ignored
using # type: ignore, we may want to treat that as imprecise. For multi-line
imports, we only report precision for the first line. These would be a bit tricky
to fix in the report generator so I decided to skip these for now.

[Also fixes a small bug with % interpolation that I encountered.]

Author: JukkaL

mypy/build.py

@@ -748,7 +748,7 @@ def report_file(self,
                     type_map: Dict[Expression, Type],
                     options: Options) -> None:
         if self.reports is not None and self.source_set.is_source(file):
-            self.reports.file(file, type_map, options)
+            self.reports.file(file, self.modules, type_map, options)
 
     def verbosity(self) -> int:
         return self.options.verbosity
@@ -2120,7 +2120,7 @@ def semantic_analysis_pass_three(self) -> None:
             self.manager.semantic_analyzer_pass3.visit_file(self.tree, self.xpath,
                                                             self.options, patches)
             if self.options.dump_type_stats:
-                dump_type_stats(self.tree, self.xpath)
+                dump_type_stats(self.tree, self.xpath, self.manager.modules)
         self.patches = patches + self.patches
 
     def semantic_analysis_apply_patches(self) -> None:
@@ -2165,7 +2165,10 @@ def finish_passes(self) -> None:
             self._patch_indirect_dependencies(self.type_checker().module_refs, self.type_map())
 
             if self.options.dump_inference_stats:
-                dump_type_stats(self.tree, self.xpath, inferred=True,
+                dump_type_stats(self.tree,
+                                self.xpath,
+                                modules=self.manager.modules,
+                                inferred=True,
                                 typemap=self.type_map())
             manager.report_file(self.tree, self.type_map(), self.options)
 

mypy/checkstrformat.py

@@ -143,6 +143,9 @@ def check_simple_str_interpolation(self, specifiers: List[ConversionSpecifier],
             rep_types = rhs_type.items
         elif isinstance(rhs_type, AnyType):
             return
+        elif isinstance(rhs_type, Instance) and rhs_type.type.fullname() == 'builtins.tuple':
+            # Assume that an arbitrary-length tuple has the right number of items.
+            rep_types = [rhs_type.args[0]] * len(checkers)
         else:
             rep_types = [rhs_type]
 

mypy/defaults.py

@@ -26,4 +26,5 @@
                   'xslt-html',
                   'xslt-txt',
                   'html',
-                  'txt']  # type: Final
+                  'txt',
+                  'lineprecision']  # type: Final

mypy/report.py

@@ -13,7 +13,7 @@
 from urllib.request import pathname2url
 
 import typing
-from typing import Any, Callable, Dict, List, Optional, Tuple, cast
+from typing import Any, Callable, Dict, List, Optional, Tuple, cast, Iterator
 from typing_extensions import Final
 
 from mypy.nodes import MypyFile, Expression, FuncDef
@@ -74,9 +74,13 @@ def add_report(self, report_type: str, report_dir: str) -> 'AbstractReporter':
         self.named_reporters[report_type] = reporter
         return reporter
 
-    def file(self, tree: MypyFile, type_map: Dict[Expression, Type], options: Options) -> None:
+    def file(self,
+             tree: MypyFile,
+             modules: Dict[str, MypyFile],
+             type_map: Dict[Expression, Type],
+             options: Options) -> None:
         for reporter in self.reporters:
-            reporter.on_file(tree, type_map, options)
+            reporter.on_file(tree, modules, type_map, options)
 
     def finish(self) -> None:
         for reporter in self.reporters:
@@ -90,7 +94,11 @@ def __init__(self, reports: Reports, output_dir: str) -> None:
             stats.ensure_dir_exists(output_dir)
 
     @abstractmethod
-    def on_file(self, tree: MypyFile, type_map: Dict[Expression, Type], options: Options) -> None:
+    def on_file(self,
+                tree: MypyFile,
+                modules: Dict[str, MypyFile],
+                type_map: Dict[Expression, Type],
+                options: Options) -> None:
         pass
 
     @abstractmethod
@@ -108,6 +116,23 @@ def alias_reporter(source_reporter: str, target_reporter: str) -> None:
     reporter_classes[target_reporter] = reporter_classes[source_reporter]
 
 
+def should_skip_path(path: str) -> bool:
+    if stats.is_special_module(path):
+        return True
+    if path.startswith('..'):
+        return True
+    if 'stubs' in path.split('/') or 'stubs' in path.split(os.sep):
+        return True
+    return False
+
+
+def iterate_python_lines(path: str) -> Iterator[Tuple[int, str]]:
+    """Return an iterator over (line number, line text) from a Python file."""
+    with tokenize.open(path) as input_file:
+        for line_info in enumerate(input_file, 1):
+            yield line_info
+
+
 class FuncCounterVisitor(TraverserVisitor):
     def __init__(self) -> None:
         super().__init__()
@@ -124,6 +149,7 @@ def __init__(self, reports: Reports, output_dir: str) -> None:
 
     def on_file(self,
                 tree: MypyFile,
+                modules: Dict[str, MypyFile],
                 type_map: Dict[Expression, Type],
                 options: Options) -> None:
         # Count physical lines.  This assumes the file's encoding is a
@@ -162,17 +188,23 @@ def on_finish(self) -> None:
 
 
 class AnyExpressionsReporter(AbstractReporter):
+    """Report frequencies of different kinds of Any types."""
+
     def __init__(self, reports: Reports, output_dir: str) -> None:
         super().__init__(reports, output_dir)
         self.counts = {}  # type: Dict[str, Tuple[int, int]]
         self.any_types_counter = {}  # type: Dict[str, typing.Counter[int]]
 
     def on_file(self,
                 tree: MypyFile,
+                modules: Dict[str, MypyFile],
                 type_map: Dict[Expression, Type],
                 options: Options) -> None:
-        visitor = stats.StatisticsVisitor(inferred=True, filename=tree.fullname(),
-                                          typemap=type_map, all_nodes=True,
+        visitor = stats.StatisticsVisitor(inferred=True,
+                                          filename=tree.fullname(),
+                                          modules=modules,
+                                          typemap=type_map,
+                                          all_nodes=True,
                                           visit_untyped_defs=False)
         tree.accept(visitor)
         self.any_types_counter[tree.fullname()] = visitor.type_of_any_counter
@@ -354,12 +386,14 @@ class LineCoverageReporter(AbstractReporter):
     source file's absolute pathname the list of line numbers that
     belong to typed functions in that file.
     """
+
     def __init__(self, reports: Reports, output_dir: str) -> None:
         super().__init__(reports, output_dir)
         self.lines_covered = {}  # type: Dict[str, List[int]]
 
     def on_file(self,
                 tree: MypyFile,
+                modules: Dict[str, MypyFile],
                 type_map: Dict[Expression, Type],
                 options: Options) -> None:
         with open(tree.path) as f:
@@ -421,34 +455,33 @@ def __init__(self, reports: Reports, output_dir: str) -> None:
 
     def on_file(self,
                 tree: MypyFile,
+                modules: Dict[str, MypyFile],
                 type_map: Dict[Expression, Type],
                 options: Options) -> None:
         self.last_xml = None
         path = os.path.relpath(tree.path)
-        if stats.is_special_module(path):
-            return
-        if path.startswith('..'):
-            return
-        if 'stubs' in path.split('/'):
+        if should_skip_path(path):
             return
 
-        visitor = stats.StatisticsVisitor(inferred=True, filename=tree.fullname(),
-                                          typemap=type_map, all_nodes=True)
+        visitor = stats.StatisticsVisitor(inferred=True,
+                                          filename=tree.fullname(),
+                                          modules=modules,
+                                          typemap=type_map,
+                                          all_nodes=True)
         tree.accept(visitor)
 
         root = etree.Element('mypy-report-file', name=path, module=tree._fullname)
         doc = etree.ElementTree(root)
         file_info = FileInfo(path, tree._fullname)
 
-        with tokenize.open(path) as input_file:
-            for lineno, line_text in enumerate(input_file, 1):
-                status = visitor.line_map.get(lineno, stats.TYPE_EMPTY)
-                file_info.counts[status] += 1
-                etree.SubElement(root, 'line',
-                                 number=str(lineno),
-                                 precision=stats.precision_names[status],
-                                 content=line_text.rstrip('\n').translate(self.control_fixer),
-                                 any_info=self._get_any_info_for_line(visitor, lineno))
+        for lineno, line_text in iterate_python_lines(path):
+            status = visitor.line_map.get(lineno, stats.TYPE_EMPTY)
+            file_info.counts[status] += 1
+            etree.SubElement(root, 'line',
+                             number=str(lineno),
+                             precision=stats.precision_names[status],
+                             content=line_text.rstrip('\n').translate(self.control_fixer),
+                             any_info=self._get_any_info_for_line(visitor, lineno))
         # Assumes a layout similar to what XmlReporter uses.
         xslt_path = os.path.relpath('mypy-html.xslt', path)
         transform_pi = etree.ProcessingInstruction('xml-stylesheet',
@@ -506,8 +539,8 @@ def get_line_rate(covered_lines: int, total_lines: int) -> str:
 
 
 class CoberturaPackage(object):
-    """Container for XML and statistics mapping python modules to Cobertura package
-    """
+    """Container for XML and statistics mapping python modules to Cobertura package."""
+
     def __init__(self, name: str) -> None:
         self.name = name
         self.classes = {}  # type: Dict[str, Any]
@@ -535,8 +568,7 @@ def add_packages(self, parent_element: Any) -> None:
 
 
 class CoberturaXmlReporter(AbstractReporter):
-    """Reporter for generating Cobertura compliant XML.
-    """
+    """Reporter for generating Cobertura compliant XML."""
 
     def __init__(self, reports: Reports, output_dir: str) -> None:
         super().__init__(reports, output_dir)
@@ -549,11 +581,15 @@ def __init__(self, reports: Reports, output_dir: str) -> None:
 
     def on_file(self,
                 tree: MypyFile,
+                modules: Dict[str, MypyFile],
                 type_map: Dict[Expression, Type],
                 options: Options) -> None:
         path = os.path.relpath(tree.path)
-        visitor = stats.StatisticsVisitor(inferred=True, filename=tree.fullname(),
-                                          typemap=type_map, all_nodes=True)
+        visitor = stats.StatisticsVisitor(inferred=True,
+                                          filename=tree.fullname(),
+                                          modules=modules,
+                                          typemap=type_map,
+                                          all_nodes=True)
         tree.accept(visitor)
 
         class_name = os.path.basename(path)
@@ -646,6 +682,7 @@ class XmlReporter(AbstractXmlReporter):
 
     def on_file(self,
                 tree: MypyFile,
+                modules: Dict[str, MypyFile],
                 type_map: Dict[Expression, Type],
                 options: Options) -> None:
         last_xml = self.memory_xml.last_xml
@@ -688,6 +725,7 @@ def __init__(self, reports: Reports, output_dir: str) -> None:
 
     def on_file(self,
                 tree: MypyFile,
+                modules: Dict[str, MypyFile],
                 type_map: Dict[Expression, Type],
                 options: Options) -> None:
         last_xml = self.memory_xml.last_xml
@@ -730,6 +768,7 @@ def __init__(self, reports: Reports, output_dir: str) -> None:
 
     def on_file(self,
                 tree: MypyFile,
+                modules: Dict[str, MypyFile],
                 type_map: Dict[Expression, Type],
                 options: Options) -> None:
         pass
@@ -749,6 +788,75 @@ def on_finish(self) -> None:
 alias_reporter('xslt-html', 'html')
 alias_reporter('xslt-txt', 'txt')
 
+
+class LinePrecisionReporter(AbstractReporter):
+    """Report per-module line counts for typing precision.
+
+    Each line is classified into one of these categories:
+
+    * precise (fully type checked)
+    * imprecise (Any types in a type component, such as List[Any])
+    * any (something with an Any type, implicit or explicit)
+    * empty (empty line, comment or docstring)
+    * unanalyzed (mypy considers line unreachable)
+
+    The meaning of these categories varies slightly depending on
+    context.
+    """
+
+    def __init__(self, reports: Reports, output_dir: str) -> None:
+        super().__init__(reports, output_dir)
+        self.files = []  # type: List[FileInfo]
+
+    def on_file(self,
+                tree: MypyFile,
+                modules: Dict[str, MypyFile],
+                type_map: Dict[Expression, Type],
+                options: Options) -> None:
+        path = os.path.relpath(tree.path)
+        if should_skip_path(path):
+            return
+
+        visitor = stats.StatisticsVisitor(inferred=True,
+                                          filename=tree.fullname(),
+                                          modules=modules,
+                                          typemap=type_map,
+                                          all_nodes=True)
+        tree.accept(visitor)
+
+        file_info = FileInfo(path, tree._fullname)
+        for lineno, _ in iterate_python_lines(path):
+            status = visitor.line_map.get(lineno, stats.TYPE_EMPTY)
+            file_info.counts[status] += 1
+
+        self.files.append(file_info)
+
+    def on_finish(self) -> None:
+        output_files = sorted(self.files, key=lambda x: x.module)
+        report_file = os.path.join(self.output_dir, 'lineprecision.txt')
+        width = max(4, max(len(info.module) for info in output_files))
+        titles = ('Lines', 'Precise', 'Imprecise', 'Any', 'Empty', 'Unanalyzed')
+        widths = (width,) + tuple(len(t) for t in titles)
+        # TODO: Need mypyc mypy pin move
+        fmt = '{:%d}  {:%d}  {:%d}  {:%d}  {:%d}  {:%d}  {:%d}\n' % widths  # type: ignore
+        with open(report_file, 'w') as f:
+            f.write(
+                fmt.format('Name', *titles))
+            f.write('-' * (width + 51) + '\n')
+            for file_info in output_files:
+                counts = file_info.counts
+                f.write(fmt.format(file_info.module.ljust(width),
+                                   file_info.total(),
+                                   counts[stats.TYPE_PRECISE],
+                                   counts[stats.TYPE_IMPRECISE],
+                                   counts[stats.TYPE_ANY],
+                                   counts[stats.TYPE_EMPTY],
+                                   counts[stats.TYPE_UNANALYZED]))
+
+
+register_reporter('lineprecision', LinePrecisionReporter)
+
+
 # Reporter class names are defined twice to speed up mypy startup, as this
 # module is slow to import. Ensure that the two definitions match.
 assert set(reporter_classes) == set(REPORTER_NAMES)