Skip to content

Commit d36a8f3

Browse files
committed
errors: speedup for large error counts
We have a legacy codebase with many errors across many files ``` Found 7995 errors in 2218 files (checked 21364 source files) ``` For historical reasons, it hasn't been practical to fix all of these yet, and we've been slowly chipping at them over time. Profiling shows that `is_blockers` is the biggest single hotspot, taking roughly 1min, and `total_errors` account for another 11s. Instead of computing those values on read by iterating over all errors, update auxiliary variables appropriately every time a new error is recorded.
1 parent 50213b5 commit d36a8f3

File tree

1 file changed

+13
-4
lines changed

1 file changed

+13
-4
lines changed

mypy/errors.py

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -134,6 +134,11 @@ class Errors:
134134
# files were processed.
135135
error_info_map: Dict[str, List[ErrorInfo]]
136136

137+
# optimization for legacy codebases with many files with errors
138+
has_real_errors: bool = False
139+
has_blockers: bool = False
140+
total_error_count: int = 0
141+
137142
# Files that we have reported the errors for
138143
flushed_files: Set[str]
139144

@@ -235,7 +240,7 @@ def copy(self) -> 'Errors':
235240
return new
236241

237242
def total_errors(self) -> int:
238-
return sum(len(errs) for errs in self.error_info_map.values())
243+
return self.total_error_count
239244

240245
def set_ignore_prefix(self, prefix: str) -> None:
241246
"""Set path prefix that will be removed from all paths."""
@@ -357,6 +362,11 @@ def _add_error_info(self, file: str, info: ErrorInfo) -> None:
357362
if file not in self.error_info_map:
358363
self.error_info_map[file] = []
359364
self.error_info_map[file].append(info)
365+
self.total_error_count += 1
366+
if info.severity == 'error':
367+
self.has_real_errors = True
368+
if info.blocker:
369+
self.has_blockers = True
360370
if info.code is IMPORT:
361371
self.seen_import_error = True
362372

@@ -553,12 +563,11 @@ def is_errors(self) -> bool:
553563

554564
def is_real_errors(self) -> bool:
555565
"""Are there any generated errors (not just notes, for example)?"""
556-
return any(info.severity == 'error'
557-
for infos in self.error_info_map.values() for info in infos)
566+
return self.has_real_errors
558567

559568
def is_blockers(self) -> bool:
560569
"""Are the any errors that are blockers?"""
561-
return any(err for errs in self.error_info_map.values() for err in errs if err.blocker)
570+
return self.has_blockers
562571

563572
def blocker_module(self) -> Optional[str]:
564573
"""Return the module with a blocking error, or None if not possible."""

0 commit comments

Comments
 (0)