Refactor calls to md5 to go through a helper (#8250)

Author: msullivan

mypy/build.py

@@ -13,7 +13,6 @@
 import contextlib
 import errno
 import gc
-import hashlib
 import json
 import os
 import pathlib
@@ -37,7 +36,7 @@
 from mypy.errors import Errors, CompileError, ErrorInfo, report_internal_error
 from mypy.util import (
     DecodeError, decode_python_encoding, is_sub_path, get_mypy_comments, module_prefix,
-    read_py_file
+    read_py_file, hash_digest,
 )
 if TYPE_CHECKING:
     from mypy.report import Reports  # Avoid unconditional slow import
@@ -468,7 +467,7 @@ def take_module_snapshot(module: types.ModuleType) -> str:
     """
     if hasattr(module, '__file__'):
         with open(module.__file__, 'rb') as f:
-            digest = hashlib.md5(f.read()).hexdigest()
+            digest = hash_digest(f.read())
     else:
         digest = 'unknown'
     ver = getattr(module, '__version__', 'none')
@@ -1262,9 +1261,9 @@ def validate_meta(meta: Optional[CacheMeta], id: str, path: Optional[str],
     # coarse-grained incremental rebuild, so we accept the cache
     # metadata even if it doesn't match the source file.
     #
-    # We still *do* the mtime/md5 checks, however, to enable
+    # We still *do* the mtime/hash checks, however, to enable
     # fine-grained mode to take advantage of the mtime-updating
-    # optimization when mtimes differ but md5s match.  There is
+    # optimization when mtimes differ but hashes match.  There is
     # essentially no extra time cost to computing the hash here, since
     # it will be cached and will be needed for finding changed files
     # later anyways.
@@ -1292,7 +1291,7 @@ def validate_meta(meta: Optional[CacheMeta], id: str, path: Optional[str],
 
         t0 = time.time()
         try:
-            source_hash = manager.fscache.md5(path)
+            source_hash = manager.fscache.hash_digest(path)
         except (OSError, UnicodeDecodeError, DecodeError):
             return None
         manager.add_stats(validate_hash_time=time.time() - t0)
@@ -1346,10 +1345,12 @@ def validate_meta(meta: Optional[CacheMeta], id: str, path: Optional[str],
 
 
 def compute_hash(text: str) -> str:
-    # We use md5 instead of the builtin hash(...) function because the output of hash(...)
-    # can differ between runs due to hash randomization (enabled by default in Python 3.3).
-    # See the note in https://docs.python.org/3/reference/datamodel.html#object.__hash__.
-    return hashlib.md5(text.encode('utf-8')).hexdigest()
+    # We use a crypto hash instead of the builtin hash(...) function
+    # because the output of hash(...)  can differ between runs due to
+    # hash randomization (enabled by default in Python 3.3).  See the
+    # note in
+    # https://docs.python.org/3/reference/datamodel.html#object.__hash__.
+    return hash_digest(text.encode('utf-8'))
 
 
 def json_dumps(obj: Any, debug_cache: bool) -> str:
@@ -1982,7 +1983,7 @@ def parse_file(self) -> None:
                     path = manager.maybe_swap_for_shadow_path(self.path)
                     source = decode_python_encoding(manager.fscache.read(path),
                                                     manager.options.python_version)
-                    self.source_hash = manager.fscache.md5(path)
+                    self.source_hash = manager.fscache.hash_digest(path)
                 except IOError as ioerr:
                     # ioerr.strerror differs for os.stat failures between Windows and
                     # other systems, but os.strerror(ioerr.errno) does not, so we use that.

mypy/dmypy_server.py

@@ -423,7 +423,7 @@ def initialize_fine_grained(self, sources: List[BuildSource],
                 assert state.path is not None
                 self.fswatcher.set_file_data(
                     state.path,
-                    FileData(st_mtime=float(meta.mtime), st_size=meta.size, md5=meta.hash))
+                    FileData(st_mtime=float(meta.mtime), st_size=meta.size, hash=meta.hash))
 
             changed, removed = self.find_changed(sources)
 

mypy/fscache.py

@@ -28,10 +28,10 @@
 advantage of the benefits.
 """
 
-import hashlib
 import os
 import stat
 from typing import Dict, List, Set
+from mypy.util import hash_digest
 
 
 class FileSystemCache:
@@ -256,12 +256,11 @@ def read(self, path: str) -> bytes:
                 self.read_error_cache[path] = err
                 raise
 
-        md5hash = hashlib.md5(data).hexdigest()
         self.read_cache[path] = data
-        self.hash_cache[path] = md5hash
+        self.hash_cache[path] = hash_digest(data)
         return data
 
-    def md5(self, path: str) -> str:
+    def hash_digest(self, path: str) -> str:
         if path not in self.hash_cache:
             self.read(path)
         return self.hash_cache[path]

mypy/fswatcher.py

@@ -6,14 +6,14 @@
 
 FileData = NamedTuple('FileData', [('st_mtime', float),
                                    ('st_size', int),
-                                   ('md5', str)])
+                                   ('hash', str)])
 
 
 class FileSystemWatcher:
     """Watcher for file system changes among specific paths.
 
     All file system access is performed using FileSystemCache. We
-    detect changed files by stat()ing them all and comparing md5 hashes
+    detect changed files by stat()ing them all and comparing hashes
     of potentially changed files. If a file has both size and mtime
     unmodified, the file is assumed to be unchanged.
 
@@ -54,8 +54,8 @@ def remove_watched_paths(self, paths: Iterable[str]) -> None:
 
     def _update(self, path: str) -> None:
         st = self.fs.stat(path)
-        md5 = self.fs.md5(path)
-        self._file_data[path] = FileData(st.st_mtime, st.st_size, md5)
+        hash_digest = self.fs.hash_digest(path)
+        self._file_data[path] = FileData(st.st_mtime, st.st_size, hash_digest)
 
     def _find_changed(self, paths: Iterable[str]) -> AbstractSet[str]:
         changed = set()
@@ -76,10 +76,10 @@ def _find_changed(self, paths: Iterable[str]) -> AbstractSet[str]:
                 # Round mtimes down, to match the mtimes we write to meta files
                 elif st.st_size != old.st_size or int(st.st_mtime) != int(old.st_mtime):
                     # Only look for changes if size or mtime has changed as an
-                    # optimization, since calculating md5 is expensive.
-                    new_md5 = self.fs.md5(path)
+                    # optimization, since calculating hash is expensive.
+                    new_hash = self.fs.hash_digest(path)
                     self._update(path)
-                    if st.st_size != old.st_size or new_md5 != old.md5:
+                    if st.st_size != old.st_size or new_hash != old.hash:
                         # Changed file.
                         changed.add(path)
         return changed

mypy/util.py

@@ -4,7 +4,7 @@
 import re
 import subprocess
 import sys
-import os
+import hashlib
 
 from typing import (
     TypeVar, List, Tuple, Optional, Dict, Sequence, Iterable, Container, IO, Callable
@@ -469,6 +469,16 @@ def soft_wrap(msg: str, max_len: int, first_offset: int,
     return padding.join(lines)
 
 
+def hash_digest(data: bytes) -> str:
+    """Compute a hash digest of some data.
+
+    We use a cryptographic hash because we want a low probability of
+    accidental collision, but we don't really care about any of the
+    cryptographic properties.
+    """
+    return hashlib.md5(data).hexdigest()
+
+
 class FancyFormatter:
     """Apply color and bold font to terminal output.
 

mypyc/emitmodule.py

@@ -4,7 +4,6 @@
 # single module and it should be renamed.
 
 import os
-import hashlib
 import json
 from collections import OrderedDict
 from typing import List, Tuple, Dict, Iterable, Set, TypeVar, Optional
@@ -18,6 +17,7 @@
 from mypy.options import Options
 from mypy.plugin import Plugin, ReportConfigContext
 from mypy.fscache import FileSystemCache
+from mypy.util import hash_digest
 
 from mypyc import genops
 from mypyc.common import (
@@ -144,7 +144,7 @@ def report_config_data(
                     contents = f.read()
             except FileNotFoundError:
                 return None
-            real_hash = hashlib.md5(contents).hexdigest()
+            real_hash = hash_digest(contents)
             if hash != real_hash:
                 return None