Initial implementation of inline configuration

Implements line configuration using '# mypy: ' comments, following the
blueprint I proposed in #2938.

It currently finds them just using a regex which means it is possible to pick
up a directive spuriously in a string literal or something but honestly I am
just not worried about that in practice.

Examples of what it looks like in the tests.

Fixes #2938.

Thoughts?

Author: msullivan

mypy/build.py

@@ -43,7 +43,7 @@
 from mypy.checker import TypeChecker
 from mypy.indirection import TypeIndirectionVisitor
 from mypy.errors import Errors, CompileError, report_internal_error
-from mypy.util import DecodeError, decode_python_encoding, is_sub_path
+from mypy.util import DecodeError, decode_python_encoding, is_sub_path, get_mypy_comments
 if MYPY:
     from mypy.report import Reports  # Avoid unconditional slow import
 from mypy import moduleinfo
@@ -61,6 +61,7 @@
 from mypy.metastore import MetadataStore, FilesystemMetadataStore, SqliteMetadataStore
 from mypy.typestate import TypeState, reset_global_state
 from mypy.renaming import VariableRenameVisitor
+from mypy.config_parser import parse_mypy_comments
 
 from mypy.mypyc_hacks import BuildManagerBase
 
@@ -1364,6 +1365,11 @@ def write_cache(id: str, path: str, tree: MypyFile,
 
     mtime = 0 if bazel else int(st.st_mtime)
     size = st.st_size
+    # Note that the options we store in the cache are the options as
+    # specified by the command line/config file and *don't* reflect
+    # updates made by inline config directives in the file. This is
+    # important, or otherwise the options would never match when
+    # verifying the cache.
     options = manager.options.clone_for_module(id)
     assert source_hash is not None
     meta = {'id': id,
@@ -1922,6 +1928,8 @@ def parse_file(self) -> None:
             else:
                 assert source is not None
                 self.source_hash = compute_hash(source)
+
+            self.parse_inline_configuration(source)
             self.tree = manager.parse_file(self.id, self.xpath, source,
                                            self.ignore_all or self.options.ignore_errors)
 
@@ -1937,6 +1945,16 @@ def parse_file(self) -> None:
 
         self.check_blockers()
 
+    def parse_inline_configuration(self, source: str) -> None:
+        # Check for inline mypy: options directive and parse them.
+        flags = get_mypy_comments(source)
+        if flags:
+            changes, config_errors = parse_mypy_comments(flags, self.options)
+            self.options = self.options.apply_changes(changes)
+            self.manager.errors.set_file(self.xpath, self.id)
+            for error in config_errors:
+                self.manager.errors.report(-1, 0, error)
+
     def semantic_analysis_pass1(self) -> None:
         """Perform pass 1 of semantic analysis, which happens immediately after parsing.
 

mypy/config_parser.py

@@ -1,6 +1,7 @@
 import argparse
 import configparser
 import glob as fileglob
+from io import StringIO
 import os
 import re
 import sys
@@ -116,24 +117,22 @@ def parse_config_file(options: Options, filename: Optional[str],
             print("%s: No [mypy] section in config file" % file_read, file=stderr)
     else:
         section = parser['mypy']
-        prefix = '%s: [%s]' % (file_read, 'mypy')
-        updates, report_dirs = parse_section(prefix, options, section,
-                                             stdout, stderr)
+        prefix = '%s: [%s]: ' % (file_read, 'mypy')
+        updates, report_dirs = parse_section(prefix, options, section, stderr)
         for k, v in updates.items():
             setattr(options, k, v)
         options.report_dirs.update(report_dirs)
 
     for name, section in parser.items():
         if name.startswith('mypy-'):
-            prefix = '%s: [%s]' % (file_read, name)
-            updates, report_dirs = parse_section(prefix, options, section,
-                                                 stdout, stderr)
+            prefix = '%s: [%s]: ' % (file_read, name)
+            updates, report_dirs = parse_section(prefix, options, section, stderr)
             if report_dirs:
-                print("%s: Per-module sections should not specify reports (%s)" %
+                print("%sPer-module sections should not specify reports (%s)" %
                       (prefix, ', '.join(s + '_report' for s in sorted(report_dirs))),
                       file=stderr)
             if set(updates) - PER_MODULE_OPTIONS:
-                print("%s: Per-module sections should only specify per-module flags (%s)" %
+                print("%sPer-module sections should only specify per-module flags (%s)" %
                       (prefix, ', '.join(sorted(set(updates) - PER_MODULE_OPTIONS))),
                       file=stderr)
                 updates = {k: v for k, v in updates.items() if k in PER_MODULE_OPTIONS}
@@ -146,7 +145,7 @@ def parse_config_file(options: Options, filename: Optional[str],
 
                 if (any(c in glob for c in '?[]!') or
                         any('*' in x and x != '*' for x in glob.split('.'))):
-                    print("%s: Patterns must be fully-qualified module names, optionally "
+                    print("%sPatterns must be fully-qualified module names, optionally "
                           "with '*' in some components (e.g spam.*.eggs.*)"
                           % prefix,
                           file=stderr)
@@ -156,7 +155,6 @@ def parse_config_file(options: Options, filename: Optional[str],
 
 def parse_section(prefix: str, template: Options,
                   section: Mapping[str, str],
-                  stdout: TextIO = sys.stdout,
                   stderr: TextIO = sys.stderr
                   ) -> Tuple[Dict[str, object], Dict[str, str]]:
     """Parse one section of a config file.
@@ -176,17 +174,17 @@ def parse_section(prefix: str, template: Options,
                     if report_type in defaults.REPORTER_NAMES:
                         report_dirs[report_type] = section[key]
                     else:
-                        print("%s: Unrecognized report type: %s" % (prefix, key),
+                        print("%sUnrecognized report type: %s" % (prefix, key),
                               file=stderr)
                     continue
                 if key.startswith('x_'):
                     continue  # Don't complain about `x_blah` flags
                 elif key == 'strict':
-                    print("%s: Strict mode is not supported in configuration files: specify "
+                    print("%sStrict mode is not supported in configuration files: specify "
                           "individual flags instead (see 'mypy -h' for the list of flags enabled "
                           "in strict mode)" % prefix, file=stderr)
                 else:
-                    print("%s: Unrecognized option: %s = %s" % (prefix, key, section[key]),
+                    print("%sUnrecognized option: %s = %s" % (prefix, key, section[key]),
                           file=stderr)
                 continue
             ct = type(dv)
@@ -198,29 +196,75 @@ def parse_section(prefix: str, template: Options,
                 try:
                     v = ct(section.get(key))
                 except argparse.ArgumentTypeError as err:
-                    print("%s: %s: %s" % (prefix, key, err), file=stderr)
+                    print("%s%s: %s" % (prefix, key, err), file=stderr)
                     continue
             else:
-                print("%s: Don't know what type %s should have" % (prefix, key), file=stderr)
+                print("%sDon't know what type %s should have" % (prefix, key), file=stderr)
                 continue
         except ValueError as err:
-            print("%s: %s: %s" % (prefix, key, err), file=stderr)
+            print("%s%s: %s" % (prefix, key, err), file=stderr)
             continue
         if key == 'cache_dir':
             v = os.path.expanduser(v)
         if key == 'silent_imports':
-            print("%s: silent_imports has been replaced by "
+            print("%ssilent_imports has been replaced by "
                   "ignore_missing_imports=True; follow_imports=skip" % prefix, file=stderr)
             if v:
                 if 'ignore_missing_imports' not in results:
                     results['ignore_missing_imports'] = True
                 if 'follow_imports' not in results:
                     results['follow_imports'] = 'skip'
         if key == 'almost_silent':
-            print("%s: almost_silent has been replaced by "
+            print("%salmost_silent has been replaced by "
                   "follow_imports=error" % prefix, file=stderr)
             if v:
                 if 'follow_imports' not in results:
                     results['follow_imports'] = 'error'
         results[key] = v
     return results, report_dirs
+
+
+def mypy_comments_to_config_map(args: List[str], template: Options) -> Dict[str, str]:
+    """Rewrite the mypy comment syntax into ini file syntax"""
+    options = {}
+    for line in args:
+        for entry in line.split(', '):
+            if '=' not in entry:
+                name = entry
+                value = None
+            else:
+                name, value = entry.split('=', 1)
+
+            name = name.replace('-', '_')
+            if value is None:
+                if name.startswith('no_') and not hasattr(template, name):
+                    name = name[3:]
+                    value = 'False'
+                else:
+                    value = 'True'
+            options[name] = value
+
+    return options
+
+
+def parse_mypy_comments(
+        args: List[str], template: Options) -> Tuple[Dict[str, object], List[str]]:
+    """Parse a collection of inline mypy: configuration comments.
+
+    Returns a dictionary of options to be applied and a list of error messages
+    generated.
+    """
+
+    # In order to easily match the behavior for bools, we abuse configparser.
+    # Oddly, the only way to get the SectionProxy object with the getboolean
+    # method is to create a config parser.
+    parser = configparser.RawConfigParser()
+    parser['dummy'] = mypy_comments_to_config_map(args, template)
+
+    stderr = StringIO()
+    sections, reports = parse_section('', template, parser['dummy'], stderr=stderr)
+    errors = [x for x in stderr.getvalue().strip().split('\n') if x]
+    if reports:
+        errors.append("Reports not supported in inline configuration")
+
+    return sections, errors

mypy/test/testcheck.py

@@ -83,6 +83,7 @@
     'check-redefine.test',
     'check-literal.test',
     'check-newsemanal.test',
+    'check-inline-config.test',
 ]
 
 # Tests that use Python 3.8-only AST features (like expression-scoped ignores):

mypy/util.py

@@ -18,6 +18,9 @@
 ENCODING_RE = \
     re.compile(br'([ \t\v]*#.*(\r\n?|\n))??[ \t\v]*#.*coding[:=][ \t]*([-\w.]+)')  # type: Final
 
+MYPY_RE = \
+    re.compile(r'^#.mypy: (.*)$', re.MULTILINE)  # type: Final
+
 default_python2_interpreter = \
     ['python2', 'python', '/usr/bin/python', 'C:\\Python27\\python.exe']  # type: Final
 
@@ -89,6 +92,10 @@ def decode_python_encoding(source: bytes, pyversion: Tuple[int, int]) -> str:
     return source_text
 
 
+def get_mypy_comments(source: str) -> List[str]:
+    return list(re.findall(MYPY_RE, source))
+
+
 _python2_interpreter = None  # type: Optional[str]
 
 

test-data/unit/check-inline-config.test

@@ -0,0 +1,108 @@
+-- Checks for 'mypy: option' directives inside files
+
+[case testInlineSimple1]
+# mypy: disallow-any-generics, no-warn-no-return
+
+from typing import List
+def foo() -> List:  # E: Missing type parameters for generic type
+    20
+
+[builtins fixtures/list.pyi]
+
+[case testInlineSimple2]
+# mypy: disallow-any-generics
+# mypy: no-warn-no-return
+
+from typing import List
+def foo() -> List:  # E: Missing type parameters for generic type
+    20
+
+[builtins fixtures/list.pyi]
+
+[case testInlineSimple3]
+# mypy: disallow-any-generics=true, warn-no-return=0
+
+from typing import List
+def foo() -> List:  # E: Missing type parameters for generic type
+    20
+
+[builtins fixtures/list.pyi]
+
+[case testInlineList]
+# mypy: disallow-any-generics, always-false=FOO,BAR
+
+from typing import List
+
+def foo(FOO: bool, BAR: bool) -> List:  # E: Missing type parameters for generic type
+    if FOO or BAR:
+        1+'lol'
+    return []
+
+[builtins fixtures/list.pyi]
+
+[case testInlineIncremental1]
+import a
+[file a.py]
+# mypy: disallow-any-generics, no-warn-no-return
+
+from typing import List
+def foo() -> List:
+    20
+
+[file a.py.2]
+# mypy: no-warn-no-return
+
+from typing import List
+def foo() -> List:
+    20
+
+[file a.py.3]
+from typing import List
+def foo() -> List:
+    20
+[out]
+tmp/a.py:4: error: Missing type parameters for generic type
+[out2]
+[out3]
+tmp/a.py:2: error: Missing return statement
+
+[builtins fixtures/list.pyi]
+
+[case testInlineIncremental2]
+# flags2: --disallow-any-generics
+import a
+[file a.py]
+# mypy: no-warn-no-return
+
+from typing import List
+def foo() -> List:
+    20
+
+[file b.py.2]
+# no changes to a.py, but flag change should cause recheck
+
+[out]
+[out2]
+tmp/a.py:4: error: Missing type parameters for generic type
+
+[builtins fixtures/list.pyi]
+
+[case testInlineIncremental3]
+import a, b
+[file a.py]
+# mypy: no-warn-no-return
+
+def foo() -> int:
+    20
+
+[file b.py]
+[file b.py.2]
+# no changes to a.py and we want to make sure it isn't rechecked
+[out]
+[out2]
+[rechecked b]
+
+[case testInlineError1]
+# mypy: invalid-whatever
+[out]
+main: error: Unrecognized option: invalid_whatever = True

test-data/unit/fine-grained.test

@@ -8845,3 +8845,39 @@ y = ''
 [out]
 ==
 ==
+
+[case testInlineConfigFineGrained1]
+import a
+[file a.py]
+# mypy: no-warn-no-return
+
+from typing import List
+def foo() -> List:
+    20
+
+[file a.py.2]
+# mypy: disallow-any-generics, no-warn-no-return
+
+from typing import List
+def foo() -> List:
+    20
+
+[file a.py.3]
+# mypy: no-warn-no-return
+
+from typing import List
+def foo() -> List:
+    20
+
+[file a.py.4]
+from typing import List
+def foo() -> List:
+    20
+[out]
+==
+a.py:4: error: Missing type parameters for generic type
+==
+==
+a.py:2: error: Missing return statement
+
+[builtins fixtures/list.pyi]