From 1dca94fbdf60aebb7f42d2e750cc46d840e65f5d Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Wed, 29 Nov 2017 18:28:09 -0800 Subject: [PATCH 1/4] Get rid of two stat() calls per file --- mypy/build.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index d716199cb993..19f926f93b86 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -20,6 +20,7 @@ import os.path import re import site +import stat import sys import time from os.path import dirname, basename @@ -950,13 +951,14 @@ def find_cache_meta(id: str, path: str, manager: BuildManager) -> Optional[Cache # TODO: May need to take more build options into account meta_json, data_json = get_cache_names(id, path, manager) manager.trace('Looking for {} at {}'.format(id, meta_json)) - if not os.path.exists(meta_json): + try: + with open(meta_json, 'r') as f: + meta_str = f.read() + manager.trace('Meta {} {}'.format(id, meta_str.rstrip())) + meta = json.loads(meta_str) # TODO: Errors + except IOError: manager.log('Could not load cache for {}: could not find {}'.format(id, meta_json)) return None - with open(meta_json, 'r') as f: - meta_str = f.read() - manager.trace('Meta {} {}'.format(id, meta_str.rstrip())) - meta = json.loads(meta_str) # TODO: Errors if not isinstance(meta, dict): manager.log('Could not load cache for {}: meta cache is not a dict: {}' .format(id, repr(meta))) @@ -1056,11 +1058,10 @@ def validate_meta(meta: Optional[CacheMeta], id: str, path: Optional[str], # TODO: Share stat() outcome with find_module() path = os.path.abspath(path) - # TODO: Don't use isfile() but check st.st_mode - if not os.path.isfile(path): + st = manager.get_stat(path) # TODO: Errors + if not stat.S_ISREG(st.st_mode): manager.log('Metadata abandoned for {}: file {} does not exist'.format(id, path)) return None - st = manager.get_stat(path) # TODO: Errors size = st.st_size if size != meta.size: manager.log('Metadata abandoned for {}: file {} has different size'.format(id, path)) From edb50151bda8d26b02769d930fa55edc650f973d Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Wed, 29 Nov 2017 18:34:14 -0800 Subject: [PATCH 2/4] Speed up clone_for_module() by using a cache (Alternatively we could put the cache in BuildManager.) --- mypy/options.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/mypy/options.py b/mypy/options.py index b8586d56f14c..1e649aa0a863 100644 --- a/mypy/options.py +++ b/mypy/options.py @@ -45,6 +45,9 @@ class Options: - {"debug_cache"}) def __init__(self) -> None: + # Cache for clone_for_module() + self.clone_cache = {} + # -- build options -- self.build_type = BuildType.STANDARD self.python_version = defaults.PYTHON3_VERSION @@ -177,6 +180,9 @@ def __repr__(self) -> str: return 'Options({})'.format(pprint.pformat(self.__dict__)) def clone_for_module(self, module: str) -> 'Options': + res = self.clone_cache.get(module) + if res is not None: + return res updates = {} for pattern in self.per_module_options: if self.module_matches_pattern(module, pattern): @@ -184,10 +190,12 @@ def clone_for_module(self, module: str) -> 'Options': del self.unused_configs[pattern] updates.update(self.per_module_options[pattern]) if not updates: + self.clone_cache[module] = self return self new_options = Options() new_options.__dict__.update(self.__dict__) new_options.__dict__.update(updates) + self.clone_cache[module] = new_options return new_options def module_matches_pattern(self, module: str, pattern: Pattern[str]) -> bool: From 11b73004b433e5dd468a2f0617f898657953915d Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Wed, 29 Nov 2017 20:41:16 -0800 Subject: [PATCH 3/4] Fix mypy --- mypy/options.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/options.py b/mypy/options.py index 1e649aa0a863..6b8d0cf08214 100644 --- a/mypy/options.py +++ b/mypy/options.py @@ -46,7 +46,7 @@ class Options: def __init__(self) -> None: # Cache for clone_for_module() - self.clone_cache = {} + self.clone_cache = {} # type: Dict[str, Options] # -- build options -- self.build_type = BuildType.STANDARD From e567b62a912810ff0b33f833f742e4cdc5c6fdb3 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Thu, 30 Nov 2017 08:45:09 -0800 Subject: [PATCH 4/4] Add docstring to clone_for_module() --- mypy/options.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/mypy/options.py b/mypy/options.py index 6b8d0cf08214..dd9bfe08c095 100644 --- a/mypy/options.py +++ b/mypy/options.py @@ -180,6 +180,11 @@ def __repr__(self) -> str: return 'Options({})'.format(pprint.pformat(self.__dict__)) def clone_for_module(self, module: str) -> 'Options': + """Create an Options object that incorporates per-module options. + + NOTE: Once this method is called all Options objects should be + considered read-only, else the caching might be incorrect. + """ res = self.clone_cache.get(module) if res is not None: return res