Add --exclude (#9992)

Resolves #4675, resolves #9981.

Additionally, we always ignore site-packages and node_modules,
and directories starting with a dot. Also note that this doesn't really 
affect import discovery; it only directly affects passing files or 
packages to mypy.

The additional check before suggesting "are you missing an
__init__.py" didn't make any sense to me, so I removed it, appended to
the message and downgraded the severity to note.

Co-authored-by: hauntsaninja <>

Author: hauntsaninja

docs/source/command_line.rst

@@ -49,6 +49,30 @@ for full details, see :ref:`running-mypy`.
     Asks mypy to type check the provided string as a program.
 
 
+.. option:: --exclude
+
+    A regular expression that matches file names, directory names and paths
+    which mypy should ignore while recursively discovering files to check.
+    Use forward slashes on all platforms.
+
+    For instance, to avoid discovering any files named `setup.py` you could
+    pass ``--exclude '/setup\.py$'``. Similarly, you can ignore discovering
+    directories with a given name by e.g. ``--exclude /build/`` or
+    those matching a subpath with ``--exclude /project/vendor/``.
+
+    Note that this flag only affects recursive discovery, that is, when mypy is
+    discovering files within a directory tree or submodules of a package to
+    check. If you pass a file or module explicitly it will still be checked. For
+    instance, ``mypy --exclude '/setup.py$' but_still_check/setup.py``.
+
+    Note that mypy will never recursively discover files and directories named
+    "site-packages", "node_modules" or "__pycache__", or those whose name starts
+    with a period, exactly as ``--exclude
+    '/(site-packages|node_modules|__pycache__|\..*)/$'`` would. Mypy will also
+    never recursively discover files with extensions other than ``.py`` or
+    ``.pyi``.
+
+
 Optional arguments
 ******************
 

docs/source/config_file.rst

@@ -197,6 +197,18 @@ section of the command line docs.
 
     This option may only be set in the global section (``[mypy]``).
 
+.. confval:: exclude
+
+    :type: regular expression
+
+    A regular expression that matches file names, directory names and paths
+    which mypy should ignore while recursively discovering files to check.
+    Use forward slashes on all platforms.
+
+    For more details, see :option:`--exclude <mypy --exclude>`.
+
+    This option may only be set in the global section (``[mypy]``).
+
 .. confval:: namespace_packages
 
     :type: boolean

docs/source/running_mypy.rst

@@ -390,7 +390,8 @@ to modules to type check.
 - Mypy will check all paths provided that correspond to files.
 
 - Mypy will recursively discover and check all files ending in ``.py`` or
-  ``.pyi`` in directory paths provided.
+  ``.pyi`` in directory paths provided, after accounting for
+  :option:`--exclude <mypy --exclude>`.
 
 - For each file to be checked, mypy will attempt to associate the file (e.g.
   ``project/foo/bar/baz.py``) with a fully qualified module name (e.g.

mypy/build.py

@@ -15,7 +15,6 @@
 import gc
 import json
 import os
-import pathlib
 import re
 import stat
 import sys
@@ -2572,6 +2571,7 @@ def log_configuration(manager: BuildManager, sources: List[BuildSource]) -> None
         ("Current Executable", sys.executable),
         ("Cache Dir", manager.options.cache_dir),
         ("Compiled", str(not __file__.endswith(".py"))),
+        ("Exclude", manager.options.exclude),
     ]
 
     for conf_name, conf_value in configuration_vars:
@@ -2771,14 +2771,12 @@ def load_graph(sources: List[BuildSource], manager: BuildManager,
                 "Duplicate module named '%s' (also at '%s')" % (st.id, graph[st.id].xpath),
                 blocker=True,
             )
-            p1 = len(pathlib.PurePath(st.xpath).parents)
-            p2 = len(pathlib.PurePath(graph[st.id].xpath).parents)
-
-            if p1 != p2:
-                manager.errors.report(
-                    -1, -1,
-                    "Are you missing an __init__.py?"
-                )
+            manager.errors.report(
+                -1, -1,
+                "Are you missing an __init__.py? Alternatively, consider using --exclude to "
+                "avoid checking one of them.",
+                severity='note'
+            )
 
             manager.errors.raise_error()
         graph[st.id] = st

mypy/find_sources.py

@@ -6,7 +6,7 @@
 from typing import List, Sequence, Set, Tuple, Optional
 from typing_extensions import Final
 
-from mypy.modulefinder import BuildSource, PYTHON_EXTENSIONS, mypy_path
+from mypy.modulefinder import BuildSource, PYTHON_EXTENSIONS, mypy_path, matches_exclude
 from mypy.fscache import FileSystemCache
 from mypy.options import Options
 
@@ -91,6 +91,8 @@ def __init__(self, fscache: FileSystemCache, options: Options) -> None:
         self.fscache = fscache
         self.explicit_package_bases = get_explicit_package_bases(options)
         self.namespace_packages = options.namespace_packages
+        self.exclude = options.exclude
+        self.verbosity = options.verbosity
 
     def is_explicit_package_base(self, path: str) -> bool:
         assert self.explicit_package_bases
@@ -103,10 +105,15 @@ def find_sources_in_dir(self, path: str) -> List[BuildSource]:
         names = sorted(self.fscache.listdir(path), key=keyfunc)
         for name in names:
             # Skip certain names altogether
-            if name == '__pycache__' or name.startswith('.') or name.endswith('~'):
+            if name in ("__pycache__", "site-packages", "node_modules") or name.startswith("."):
                 continue
             subpath = os.path.join(path, name)
 
+            if matches_exclude(
+                subpath, self.exclude, self.fscache, self.verbosity >= 2
+            ):
+                continue
+
             if self.fscache.isdir(subpath):
                 sub_sources = self.find_sources_in_dir(subpath)
                 if sub_sources:

mypy/main.py

@@ -808,6 +808,15 @@ def add_invertible_flag(flag: str,
     code_group.add_argument(
         '--explicit-package-bases', action='store_true',
         help="Use current directory and MYPYPATH to determine module names of files passed")
+    code_group.add_argument(
+        "--exclude",
+        metavar="PATTERN",
+        default="",
+        help=(
+            "Regular expression to match file names, directory names or paths which mypy should "
+            "ignore while recursively discovering files to check, e.g. --exclude '/setup\\.py$'"
+        )
+    )
     code_group.add_argument(
         '-m', '--module', action='append', metavar='MODULE',
         default=[],

mypy/modulefinder.py

@@ -7,6 +7,7 @@
 import collections
 import functools
 import os
+import re
 import subprocess
 import sys
 from enum import Enum
@@ -443,10 +444,15 @@ def find_modules_recursive(self, module: str) -> List[BuildSource]:
         names = sorted(self.fscache.listdir(package_path))
         for name in names:
             # Skip certain names altogether
-            if name == '__pycache__' or name.startswith('.') or name.endswith('~'):
+            if name in ("__pycache__", "site-packages", "node_modules") or name.startswith("."):
                 continue
             subpath = os.path.join(package_path, name)
 
+            if self.options and matches_exclude(
+                subpath, self.options.exclude, self.fscache, self.options.verbosity >= 2
+            ):
+                continue
+
             if self.fscache.isdir(subpath):
                 # Only recurse into packages
                 if (self.options and self.options.namespace_packages) or (
@@ -460,13 +466,26 @@ def find_modules_recursive(self, module: str) -> List[BuildSource]:
                 if stem == '__init__':
                     continue
                 if stem not in seen and '.' not in stem and suffix in PYTHON_EXTENSIONS:
-                    # (If we sorted names) we could probably just make the BuildSource ourselves,
-                    # but this ensures compatibility with find_module / the cache
+                    # (If we sorted names by keyfunc) we could probably just make the BuildSource
+                    # ourselves, but this ensures compatibility with find_module / the cache
                     seen.add(stem)
                     sources.extend(self.find_modules_recursive(module + '.' + stem))
         return sources
 
 
+def matches_exclude(subpath: str, exclude: str, fscache: FileSystemCache, verbose: bool) -> bool:
+    if not exclude:
+        return False
+    subpath_str = os.path.abspath(subpath).replace(os.sep, "/")
+    if fscache.isdir(subpath):
+        subpath_str += "/"
+    if re.search(exclude, subpath_str):
+        if verbose:
+            print("TRACE: Excluding {}".format(subpath_str), file=sys.stderr)
+        return True
+    return False
+
+
 def verify_module(fscache: FileSystemCache, id: str, path: str, prefix: str) -> bool:
     """Check that all packages containing id have a __init__ file."""
     if path.endswith(('__init__.py', '__init__.pyi')):

mypy/options.py

@@ -97,6 +97,8 @@ def __init__(self) -> None:
         # sufficient to determine module names for files. As a possible alternative, add a single
         # top-level __init__.py to your packages.
         self.explicit_package_bases = False
+        # File names, directory names or subpaths to avoid checking
+        self.exclude = ""  # type: str
 
         # disallow_any options
         self.disallow_any_generics = False