Rework (and heavily optimize!) mypy.ini per-module configuration

docs/source/config_file.rst

@@ -29,10 +29,15 @@ characters.
   the global flags. The ``setup.cfg`` file is an exception to this.
 
 - Additional sections named ``[mypy-PATTERN1,PATTERN2,...]`` may be
-  present, where ``PATTERN1``, ``PATTERN2`` etc. are `fnmatch patterns
-  <https://docs.python.org/3.6/library/fnmatch.html>`_
-  separated by commas.  These sections specify additional flags that
-  only apply to *modules* whose name matches at least one of the patterns.
+  present, where ``PATTERN1``, ``PATTERN2``, etc., are comma-separated
+  patterns of the form ``dotted_module_name`` or ``dotted_module_name.*``.
+  These sections specify additional flags that only apply to *modules*
+  whose name matches at least one of the patterns.
+
+  A pattern of the form ``dotted_module_name`` matches only the named module,
+  while ``dotted_module_name.*`` matches ``dotted_module_name`` and any
+  submodules (so ``foo.bar.*`` would match all of ``foo.bar``,
+  ``foo.bar.baz``, and ``foo.bar.baz.quux``).
 
 .. note::
 
@@ -137,8 +142,12 @@ overridden by the pattern sections matching the module name.
 
 .. note::
 
-   If multiple pattern sections match a module they are processed in
-   order of their occurrence in the config file.
+   If multiple pattern sections match a module, the options from the
+   most specific section are used where they disagree.  This means
+   that ``foo.bar`` will take values from sections with the patterns
+   ``foo.bar``, ``foo.bar.*``, and ``foo.*``, but when they specify
+   different values, it will use values from ``foo.bar`` before
+   ``foo.bar.*`` before ``foo.*``.
 
 - ``follow_imports`` (string, default ``normal``) directs what to do
   with imports when the imported module is found as a ``.py`` file and

mypy/main.py

@@ -3,7 +3,6 @@
 import argparse
 import ast
 import configparser
-import fnmatch
 import os
 import re
 import subprocess
@@ -94,7 +93,8 @@ def flush_errors(new_messages: List[str], serious: bool) -> None:
     if options.warn_unused_configs and options.unused_configs:
         print("Warning: unused section(s) in %s: %s" %
               (options.config_file,
-               ", ".join("[mypy-%s]" % glob for glob in options.unused_configs.values())),
+               ", ".join("[mypy-%s]" % glob for glob in options.per_module_options.keys()
+                         if glob in options.unused_configs)),
               file=sys.stderr)
     if options.junit_xml:
         t1 = time.time()
@@ -739,9 +739,14 @@ def parse_config_file(options: Options, filename: Optional[str]) -> None:
                 glob = glob.replace(os.sep, '.')
                 if os.altsep:
                     glob = glob.replace(os.altsep, '.')
-                pattern = re.compile(fnmatch.translate(glob))
-                options.per_module_options[pattern] = updates
-                options.unused_configs[pattern] = glob
+
+                if (any(c in glob for c in '?[]!') or
+                        ('*' in glob and (not glob.endswith('.*') or '*' in glob[:-2]))):
+                    print("%s: Invalid pattern. Patterns must be 'module_name' or 'module_name.*'"
+                          % prefix,
+                          file=sys.stderr)
+                else:
+                    options.per_module_options[glob] = updates
 
 
 def parse_section(prefix: str, template: Options,

mypy/options.py

@@ -1,9 +1,8 @@
 from collections import OrderedDict
-import fnmatch
 import pprint
 import sys
 
-from typing import Dict, List, Mapping, MutableMapping, Optional, Pattern, Set, Tuple
+from typing import Dict, List, Mapping, MutableMapping, Optional, Set, Tuple
 
 from mypy import defaults
 
@@ -51,7 +50,7 @@ class Options:
 
     def __init__(self) -> None:
         # Cache for clone_for_module()
-        self.clone_cache = {}  # type: Dict[str, Options]
+        self.per_module_cache = None  # type: Optional[Dict[str, Options]]
 
         # -- build options --
         self.build_type = BuildType.STANDARD
@@ -167,10 +166,9 @@ def __init__(self) -> None:
         self.plugins = []  # type: List[str]
 
         # Per-module options (raw)
-        pm_opts = OrderedDict()  # type: OrderedDict[Pattern[str], Dict[str, object]]
+        pm_opts = OrderedDict()  # type: OrderedDict[str, Dict[str, object]]
         self.per_module_options = pm_opts
-        # Map pattern back to glob
-        self.unused_configs = OrderedDict()  # type: OrderedDict[Pattern[str], str]
+        self.unused_configs = set()  # type: Set[str]
 
         # -- development options --
         self.verbosity = 0  # More verbose messages (for troubleshooting)
@@ -202,38 +200,63 @@ def __ne__(self, other: object) -> bool:
 
     def __repr__(self) -> str:
         d = dict(self.__dict__)
-        del d['clone_cache']
+        del d['per_module_cache']
         return 'Options({})'.format(pprint.pformat(d))
 
+    def build_per_module_cache(self) -> None:
+        self.per_module_cache = {}
+        # Since configs inherit from glob configs above them in the hierarchy,
+        # we need to process per-module configs in a careful order.
+        # We have to process foo.* before foo.bar.* before foo.bar.
+        # To do this, process all glob configs before non-glob configs and
+        # exploit the fact that foo.* sorts earlier ASCIIbetically (unicodebetically?)
+        # than foo.bar.*.
+        keys = (sorted(k for k in self.per_module_options.keys() if k.endswith('.*')) +
+                [k for k in self.per_module_options.keys() if not k.endswith('.*')])
+        for key in keys:
+            # Find what the options for this key would be, just based
+            # on inheriting from parent configs.
+            options = self.clone_for_module(key)
+            # And then update it with its per-module options.
+            new_options = Options()
+            new_options.__dict__.update(options.__dict__)
+            new_options.__dict__.update(self.per_module_options[key])
+            self.per_module_cache[key] = new_options
+
+        self.unused_configs = set(keys)
+
     def clone_for_module(self, module: str) -> 'Options':
         """Create an Options object that incorporates per-module options.
 
         NOTE: Once this method is called all Options objects should be
         considered read-only, else the caching might be incorrect.
         """
-        res = self.clone_cache.get(module)
-        if res is not None:
-            return res
-        updates = {}
-        for pattern in self.per_module_options:
-            if self.module_matches_pattern(module, pattern):
-                if pattern in self.unused_configs:
-                    del self.unused_configs[pattern]
-                updates.update(self.per_module_options[pattern])
-        if not updates:
-            self.clone_cache[module] = self
-            return self
-        new_options = Options()
-        new_options.__dict__.update(self.__dict__)
-        new_options.__dict__.update(updates)
-        self.clone_cache[module] = new_options
-        return new_options
-
-    def module_matches_pattern(self, module: str, pattern: Pattern[str]) -> bool:
-        # If the pattern is 'mod.*', we want 'mod' to match that too.
-        # (That's so that a pattern specifying a package also matches
-        # that package's __init__.)
-        return pattern.match(module) is not None or pattern.match(module + '.') is not None
+        if self.per_module_cache is None:
+            self.build_per_module_cache()
+            assert self.per_module_cache is not None
+
+        # If the module just directly has a config entry, use it.
+        if module in self.per_module_cache:
+            self.unused_configs.discard(module)
+            return self.per_module_cache[module]
+
+        # If not, search for glob paths at all the parents. So if we are looking for
+        # options for foo.bar.baz, we search foo.bar.baz.*, foo.bar.*, foo.*,
+        # in that order, looking for an entry.
+        # This is technically quadratic in the length of the path, but module paths
+        # don't actually get all that long.
+        path = module.split('.')
+        for i in range(len(path), 0, -1):
+            key = '.'.join(path[:i] + ['*'])
+            if key in self.per_module_cache:
+                self.unused_configs.discard(key)
+                return self.per_module_cache[key]
+
+        # We could update the cache to directly point to modules once
+        # they have been looked up, but in testing this made things
+        # slower and not faster, so we don't bother.
+
+        return self
 
     def select_options_affecting_cache(self) -> Mapping[str, object]:
         return {opt: getattr(self, opt) for opt in self.OPTIONS_AFFECTING_CACHE}