Reset cache on mypy change (#1952)

* Make modify invalidate cache if version id changes

This commit modifies build.py to add an extra "version_id" attribute to
the cache metadata. If the version id specified in mypy/version.py
differs from the id stored in the cached data, mypy will automatically
invalidate that cached data.

* Include git hash in version.py during setup

This commit modifies setup.py to automatically edit the installed
mypy/version.py file during installation to include the git hash if
applicable.

If setup.py is not being run from a git repo, or if a git executable
can't be found in the system, mypy will default to using the original
version id.

Author: Michael0x2a

mypy/build.py

@@ -137,7 +137,7 @@ def build(sources: List[BuildSource],
         # Use stub builtins (to speed up test cases and to make them easier to
         # debug).  This is a test-only feature, so assume our files are laid out
         # as in the source tree.
-        root_dir = os.path.dirname(os.path.dirname(__file__))
+        root_dir = dirname(dirname(__file__))
         lib_path.insert(0, os.path.join(root_dir, 'test-data', 'unit', 'lib-stub'))
     else:
         for source in sources:
@@ -172,7 +172,9 @@ def build(sources: List[BuildSource],
                            ignore_prefix=os.getcwd(),
                            source_set=source_set,
                            reports=reports,
-                           options=options)
+                           options=options,
+                           version_id=__version__,
+                           )
 
     try:
         dispatch(sources, manager)
@@ -285,6 +287,7 @@ def default_lib_path(data_dir: str, pyversion: Tuple[int, int]) -> List[str]:
                         ('child_modules', List[str]),  # all submodules of the given module
                         ('options', Optional[Dict[str, bool]]),  # build options
                         ('dep_prios', List[int]),
+                        ('version_id', str),  # mypy version for cache invalidation
                         ])
 # NOTE: dependencies + suppressed == all reachable imports;
 # suppressed contains those reachable imports that were prevented by
@@ -320,14 +323,16 @@ class BuildManager:
       options:         Build options
       missing_modules: Set of modules that could not be imported encountered so far
       stale_modules:   Set of modules that needed to be rechecked
+      version_id:      The current mypy version (based on commit id when possible)
     """
 
     def __init__(self, data_dir: str,
                  lib_path: List[str],
                  ignore_prefix: str,
                  source_set: BuildSourceSet,
                  reports: Reports,
-                 options: Options) -> None:
+                 options: Options,
+                 version_id: str) -> None:
         self.start_time = time.time()
         self.data_dir = data_dir
         self.errors = Errors(options.suppress_error_context)
@@ -336,6 +341,7 @@ def __init__(self, data_dir: str,
         self.source_set = source_set
         self.reports = reports
         self.options = options
+        self.version_id = version_id
         self.semantic_analyzer = SemanticAnalyzer(lib_path, self.errors, options=options)
         self.modules = self.semantic_analyzer.modules
         self.semantic_analyzer_pass3 = ThirdPass(self.modules, self.errors)
@@ -717,14 +723,17 @@ def find_cache_meta(id: str, path: str, manager: BuildManager) -> Optional[Cache
         meta.get('child_modules', []),
         meta.get('options'),
         meta.get('dep_prios', []),
+        meta.get('version_id'),
     )
     if (m.id != id or m.path != path or
             m.mtime is None or m.size is None or
             m.dependencies is None or m.data_mtime is None):
         return None
 
     # Ignore cache if generated by an older mypy version.
-    if m.options is None or len(m.dependencies) != len(m.dep_prios):
+    if (m.version_id != manager.version_id
+            or m.options is None
+            or len(m.dependencies) != len(m.dep_prios)):
         return None
 
     # Ignore cache if (relevant) options aren't the same.
@@ -810,6 +819,7 @@ def write_cache(id: str, path: str, tree: MypyFile,
             'child_modules': child_modules,
             'options': select_options_affecting_cache(manager.options),
             'dep_prios': dep_prios,
+            'version_id': manager.version_id,
             }
     with open(meta_json_tmp, 'w') as f:
         json.dump(meta, f, sort_keys=True)

mypy/test/testgraph.py

@@ -5,6 +5,7 @@
 from mypy.myunit import Suite, assert_equal
 from mypy.build import BuildManager, State
 from mypy.build import topsort, strongly_connected_components, sorted_components, order_ascc
+from mypy.version import __version__
 from mypy.options import Options
 
 
@@ -38,7 +39,9 @@ def _make_manager(self):
             ignore_prefix='',
             source_set=None,
             reports=None,
-            options=Options())
+            options=Options(),
+            version_id=__version__,
+        )
         return manager
 
     def test_sorted_components(self) -> None:

setup.py

@@ -10,6 +10,7 @@
     exit(1)
 
 from distutils.core import setup
+from distutils.command.build_py import build_py
 from mypy.version import __version__
 from mypy import git
 
@@ -30,6 +31,19 @@
 '''.lstrip()
 
 
+def cache_version_id():
+    """Returns the version id to use for the incremental hash.
+
+    If setup.py is run from a git repo, the git commit hash will be
+    included if possible. If not, then this function will fall back to
+    using the default version id from mypy/version.py."""
+    if git.is_git_repo('.') and git.have_git():
+        return __version__ + '-' + git.git_revision('.').decode('utf-8')
+    else:
+        # Default fallback
+        return __version__
+
+
 def find_data_files(base, globs):
     """Find all interesting data files, for setup(data_files=)
 
@@ -51,6 +65,19 @@ def find_data_files(base, globs):
 
     return rv
 
+
+class CustomPythonBuild(build_py):
+    def pin_version(self):
+        path = os.path.join(self.build_lib, 'mypy')
+        self.mkpath(path)
+        with open(os.path.join(path, 'version.py'), 'w') as stream:
+            stream.write('__version__ = "{}"\n'.format(cache_version_id()))
+
+    def run(self):
+        self.execute(self.pin_version, ())
+        build_py.run(self)
+
+
 data_files = []
 
 data_files += find_data_files('typeshed', ['*.py', '*.pyi'])
@@ -93,4 +120,5 @@ def find_data_files(base, globs):
       scripts=scripts,
       data_files=data_files,
       classifiers=classifiers,
+      cmdclass={'build_py': CustomPythonBuild},
       )