Restrict the number of errors shown when there are missing stubs (#10579)

When upgrading to mypy 0.900, most projects will see some errors about
missing stubs. Projects with strict settings could see thousands of errors,
since missing stubs will generate many additional Any types.

After 200 errors (only if some of them are about imports) we will now 
only show errors about unresolved imports or missing stubs, so that the 
likely root causes won't be obscured in a high volume of errors.

Fixes #10529.

Author: JukkaL

docs/source/command_line.rst

@@ -687,6 +687,14 @@ in error messages.
 
     Show absolute paths to files.
 
+.. option:: --soft-error-limit N
+
+    This flag will adjust the limit after which mypy will (sometimes)
+    disable reporting most additional errors. The limit only applies
+    if it seems likely that most of the remaining errors will not be
+    useful or they may be overly noisy. If ``N`` is negative, there is
+    no limit. The default limit is 200.
+
 
 .. _incremental:
 

mypy/build.py

@@ -224,7 +224,8 @@ def _build(sources: List[BuildSource],
                     lambda path: read_py_file(path, cached_read, options.python_version),
                     options.show_absolute_path,
                     options.enabled_error_codes,
-                    options.disabled_error_codes)
+                    options.disabled_error_codes,
+                    options.many_errors_threshold)
     plugin, snapshot = load_plugins(options, errors, stdout, extra_plugins)
 
     # Add catch-all .gitignore to cache dir if we created it

mypy/defaults.py

@@ -30,3 +30,7 @@
                   'html',
                   'txt',
                   'lineprecision']  # type: Final
+
+# Threshold after which we sometimes filter out most errors to avoid very
+# verbose output
+MANY_ERRORS_THRESHOLD = 200  # type: Final

mypy/errors.py

@@ -10,7 +10,7 @@
 from mypy.scope import Scope
 from mypy.options import Options
 from mypy.version import __version__ as mypy_version
-from mypy.errorcodes import ErrorCode
+from mypy.errorcodes import ErrorCode, IMPORT
 from mypy import errorcodes as codes
 from mypy.util import DEFAULT_SOURCE_OFFSET, is_typeshed_file
 
@@ -65,6 +65,10 @@ class ErrorInfo:
     # Fine-grained incremental target where this was reported
     target = None  # type: Optional[str]
 
+    # If True, don't show this message in output, but still record the error (needed
+    # by mypy daemon)
+    hidden = False
+
     def __init__(self,
                  import_ctx: List[Tuple[str, int]],
                  file: str,
@@ -158,6 +162,10 @@ class Errors:
     target_module = None  # type: Optional[str]
     scope = None  # type: Optional[Scope]
 
+    # Have we seen an import-related error so far? If yes, we filter out other messages
+    # in some cases to avoid reporting huge numbers of errors.
+    seen_import_error = False
+
     def __init__(self,
                  show_error_context: bool = False,
                  show_column_numbers: bool = False,
@@ -166,7 +174,8 @@ def __init__(self,
                  read_source: Optional[Callable[[str], Optional[List[str]]]] = None,
                  show_absolute_path: bool = False,
                  enabled_error_codes: Optional[Set[ErrorCode]] = None,
-                 disabled_error_codes: Optional[Set[ErrorCode]] = None) -> None:
+                 disabled_error_codes: Optional[Set[ErrorCode]] = None,
+                 many_errors_threshold: int = -1) -> None:
         self.show_error_context = show_error_context
         self.show_column_numbers = show_column_numbers
         self.show_error_codes = show_error_codes
@@ -176,6 +185,7 @@ def __init__(self,
         self.read_source = read_source
         self.enabled_error_codes = enabled_error_codes or set()
         self.disabled_error_codes = disabled_error_codes or set()
+        self.many_errors_threshold = many_errors_threshold
         self.initialize()
 
     def initialize(self) -> None:
@@ -189,6 +199,7 @@ def initialize(self) -> None:
         self.only_once_messages = set()
         self.scope = None
         self.target_module = None
+        self.seen_import_error = False
 
     def reset(self) -> None:
         self.initialize()
@@ -201,12 +212,14 @@ def copy(self) -> 'Errors':
                      self.read_source,
                      self.show_absolute_path,
                      self.enabled_error_codes,
-                     self.disabled_error_codes)
+                     self.disabled_error_codes,
+                     self.many_errors_threshold)
         new.file = self.file
         new.import_ctx = self.import_ctx[:]
         new.function_or_member = self.function_or_member[:]
         new.target_module = self.target_module
         new.scope = self.scope
+        new.seen_import_error = self.seen_import_error
         return new
 
     def total_errors(self) -> int:
@@ -330,6 +343,8 @@ def _add_error_info(self, file: str, info: ErrorInfo) -> None:
         if file not in self.error_info_map:
             self.error_info_map[file] = []
         self.error_info_map[file].append(info)
+        if info.code is IMPORT:
+            self.seen_import_error = True
 
     def add_error_info(self, info: ErrorInfo) -> None:
         file, line, end_line = info.origin
@@ -354,8 +369,52 @@ def add_error_info(self, info: ErrorInfo) -> None:
             if info.message in self.only_once_messages:
                 return
             self.only_once_messages.add(info.message)
+        if self.seen_import_error and info.code is not IMPORT and self.has_many_errors():
+            # Missing stubs can easily cause thousands of errors about
+            # Any types, especially when upgrading to mypy 0.900,
+            # which no longer bundles third-party library stubs. Avoid
+            # showing too many errors to make it easier to see
+            # import-related errors.
+            info.hidden = True
+            self.report_hidden_errors(info)
         self._add_error_info(file, info)
 
+    def has_many_errors(self) -> bool:
+        if self.many_errors_threshold < 0:
+            return False
+        if len(self.error_info_map) >= self.many_errors_threshold:
+            return True
+        if sum(len(errors)
+               for errors in self.error_info_map.values()) >= self.many_errors_threshold:
+            return True
+        return False
+
+    def report_hidden_errors(self, info: ErrorInfo) -> None:
+        message = (
+            '(Skipping most remaining errors due to unresolved imports or missing stubs; ' +
+            'fix these first)'
+        )
+        if message in self.only_once_messages:
+            return
+        self.only_once_messages.add(message)
+        new_info = ErrorInfo(
+            import_ctx=info.import_ctx,
+            file=info.file,
+            module=info.module,
+            typ=None,
+            function_or_member=None,
+            line=info.line,
+            column=info.line,
+            severity='note',
+            message=message,
+            code=None,
+            blocker=False,
+            only_once=True,
+            origin=info.origin,
+            target=info.target,
+        )
+        self._add_error_info(info.origin[0], new_info)
+
     def is_ignored_error(self, line: int, info: ErrorInfo, ignores: Dict[int, List[str]]) -> bool:
         if info.blocker:
             # Blocking errors can never be ignored
@@ -453,6 +512,7 @@ def format_messages(self, error_info: List[ErrorInfo],
         severity 'error').
         """
         a = []  # type: List[str]
+        error_info = [info for info in error_info if not info.hidden]
         errors = self.render_messages(self.sort_messages(error_info))
         errors = self.remove_duplicates(errors)
         for file, line, column, severity, message, code in errors:

mypy/main.py

@@ -663,6 +663,8 @@ def add_invertible_flag(flag: str,
     add_invertible_flag('--show-absolute-path', default=False,
                         help="Show absolute paths to files",
                         group=error_group)
+    error_group.add_argument('--soft-error-limit', default=defaults.MANY_ERRORS_THRESHOLD,
+                             type=int, dest="many_errors_threshold", help=argparse.SUPPRESS)
 
     incremental_group = parser.add_argument_group(
         title='Incremental mode',

mypy/options.py

@@ -295,6 +295,10 @@ def __init__(self) -> None:
         self.show_absolute_path = False  # type: bool
         # Install missing stub packages if True
         self.install_types = False
+        # When we encounter errors that may cause many additional errors,
+        # skip most errors after this many messages have been reported.
+        # -1 means unlimited.
+        self.many_errors_threshold = defaults.MANY_ERRORS_THRESHOLD
 
     # To avoid breaking plugin compatibility, keep providing new_semantic_analyzer
     @property

test-data/unit/check-modules.test

@@ -2982,3 +2982,85 @@ T = TypeVar("T")
 class F(M):
     x: C
     class C: ...
+
+[case testLimitLegacyStubErrorVolume]
+# flags: --disallow-any-expr --soft-error-limit=5
+import certifi # E: Cannot find implementation or library stub for module named "certifi" \
+               # N: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # N: (Skipping most remaining errors due to unresolved imports or missing stubs; fix these first)
+certifi.x
+certifi.x
+certifi.x
+certifi.x
+
+[case testDoNotLimitErrorVolumeIfNotImportErrors]
+# flags: --disallow-any-expr --soft-error-limit=5
+def f(): pass
+certifi = f()  # E: Expression has type "Any"
+1()  # E: "int" not callable
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+1()  # E: "int" not callable
+
+
+[case testDoNotLimitImportErrorVolume]
+# flags: --disallow-any-expr --soft-error-limit=3
+import xyz1  # E: Cannot find implementation or library stub for module named "xyz1" \
+             # N: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports
+import xyz2  # E: Cannot find implementation or library stub for module named "xyz2"
+import xyz3  # E: Cannot find implementation or library stub for module named "xyz3"
+import xyz4  # E: Cannot find implementation or library stub for module named "xyz4"
+
+[case testUnlimitedStubErrorVolume]
+# flags: --disallow-any-expr --soft-error-limit=-1
+import certifi # E: Cannot find implementation or library stub for module named "certifi" \
+               # N: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"