Two optimizations for load_graph() (#4294)

Profiling load_graph() found that there were some unnecessary stat() calls going on, and, more importantly, that Options.clone_for_module() is very inefficient if you have many sections in your mypy.ini file. I solved the latter by introducing a cache. An alternative design would move the cache into BuildManager -- that would avoid a reference cycle, but it turns out that having the cache in the root Options object means that it survives between dmypy check runs, and that's a nice win there. (This explains at least partly why load_graph() was slow even with everything loaded in memory.)

Author: gvanrossum

mypy/build.py

@@ -20,6 +20,7 @@
 import os.path
 import re
 import site
+import stat
 import sys
 import time
 from os.path import dirname, basename
@@ -950,13 +951,14 @@ def find_cache_meta(id: str, path: str, manager: BuildManager) -> Optional[Cache
     # TODO: May need to take more build options into account
     meta_json, data_json = get_cache_names(id, path, manager)
     manager.trace('Looking for {} at {}'.format(id, meta_json))
-    if not os.path.exists(meta_json):
+    try:
+        with open(meta_json, 'r') as f:
+            meta_str = f.read()
+            manager.trace('Meta {} {}'.format(id, meta_str.rstrip()))
+            meta = json.loads(meta_str)  # TODO: Errors
+    except IOError:
         manager.log('Could not load cache for {}: could not find {}'.format(id, meta_json))
         return None
-    with open(meta_json, 'r') as f:
-        meta_str = f.read()
-        manager.trace('Meta {} {}'.format(id, meta_str.rstrip()))
-        meta = json.loads(meta_str)  # TODO: Errors
     if not isinstance(meta, dict):
         manager.log('Could not load cache for {}: meta cache is not a dict: {}'
                     .format(id, repr(meta)))
@@ -1056,11 +1058,10 @@ def validate_meta(meta: Optional[CacheMeta], id: str, path: Optional[str],
 
     # TODO: Share stat() outcome with find_module()
     path = os.path.abspath(path)
-    # TODO: Don't use isfile() but check st.st_mode
-    if not os.path.isfile(path):
+    st = manager.get_stat(path)  # TODO: Errors
+    if not stat.S_ISREG(st.st_mode):
         manager.log('Metadata abandoned for {}: file {} does not exist'.format(id, path))
         return None
-    st = manager.get_stat(path)  # TODO: Errors
     size = st.st_size
     if size != meta.size:
         manager.log('Metadata abandoned for {}: file {} has different size'.format(id, path))

mypy/options.py

@@ -45,6 +45,9 @@ class Options:
                                - {"debug_cache"})
 
     def __init__(self) -> None:
+        # Cache for clone_for_module()
+        self.clone_cache = {}  # type: Dict[str, Options]
+
         # -- build options --
         self.build_type = BuildType.STANDARD
         self.python_version = defaults.PYTHON3_VERSION
@@ -177,17 +180,27 @@ def __repr__(self) -> str:
         return 'Options({})'.format(pprint.pformat(self.__dict__))
 
     def clone_for_module(self, module: str) -> 'Options':
+        """Create an Options object that incorporates per-module options.
+
+        NOTE: Once this method is called all Options objects should be
+        considered read-only, else the caching might be incorrect.
+        """
+        res = self.clone_cache.get(module)
+        if res is not None:
+            return res
         updates = {}
         for pattern in self.per_module_options:
             if self.module_matches_pattern(module, pattern):
                 if pattern in self.unused_configs:
                     del self.unused_configs[pattern]
                 updates.update(self.per_module_options[pattern])
         if not updates:
+            self.clone_cache[module] = self
             return self
         new_options = Options()
         new_options.__dict__.update(self.__dict__)
         new_options.__dict__.update(updates)
+        self.clone_cache[module] = new_options
         return new_options
 
     def module_matches_pattern(self, module: str, pattern: Pattern[str]) -> bool: