Support reading config from pyproject.toml

pslacerda · pslacerda · commit 26b5754fb984 · 2020-02-29T18:16:18.000-03:00
diff --git a/mypy/config_parser.py b/mypy/config_parser.py
@@ -6,7 +6,8 @@
 import re
 import sys
 
-from typing import Any, Callable, Dict, List, Mapping, Optional, Tuple, TextIO
+import toml
+from typing import Any, Callable, Dict, List, Mapping, Optional, Tuple, TextIO, MutableMapping
 from typing_extensions import Final
 
 from mypy import defaults
@@ -48,11 +49,22 @@ def split_and_match_files(paths: str) -> List[str]:
 
     Where a path/glob matches no file, we still include the raw path in the resulting list.
 
+    Returns a list of file paths
+    """
+
+    return match_files(paths.split(','))
+
+
+def match_files(paths: List[str]) -> List[str]:
+    """Take list of files/directories (with support for globbing through the glob library).
+
+    Where a path/glob matches no file, we still include the raw path in the resulting list.
+
     Returns a list of file paths
     """
     expanded_paths = []
 
-    for path in paths.split(','):
+    for path in paths:
         path = expand_path(path.strip())
         globbed_files = fileglob.glob(path, recursive=True)
         if globbed_files:
@@ -67,7 +79,7 @@ def split_and_match_files(paths: str) -> List[str]:
 # sufficient, and we don't have to do anything here.  This table
 # exists to specify types for values initialized to None or container
 # types.
-config_types = {
+ini_type_converters = {
     'python_version': parse_version,
     'strict_optional_whitelist': lambda s: s.split(),
     'custom_typing_module': str,
@@ -88,6 +100,16 @@ def split_and_match_files(paths: str) -> List[str]:
 }  # type: Final
 
 
+toml_type_converters = {
+    'python_version': parse_version,
+    'custom_typeshed_dir': expand_path,
+    'mypy_path': lambda l: [expand_path(p) for p in l],
+    'files': match_files,
+    'cache_dir': expand_path,
+    'python_executable': expand_path,
+}  # type: Final
+
+
 def parse_config_file(options: Options, set_strict_flags: Callable[[], None],
                       filename: Optional[str],
                       stdout: Optional[TextIO] = None,
@@ -98,45 +120,130 @@ def parse_config_file(options: Options, set_strict_flags: Callable[[], None],
 
     If filename is None, fall back to default config files.
     """
-    stdout = stdout or sys.stdout
+    if filename is not None:
+        filename = os.path.expanduser(filename)
+        if os.path.splitext(filename)[1] == '.toml':
+            parse_toml_config_file(options, set_strict_flags, filename, stdout, stderr)
+        else:
+            parse_ini_config_file(options, set_strict_flags, filename, stdout, stderr)
+    else:
+        for filename in defaults.CONFIG_FILES:
+            filename = os.path.expanduser(filename)
+            if not os.path.isfile(filename):
+                continue
+            if os.path.splitext(filename)[1] == '.toml':
+                parsed = parse_toml_config_file(options, set_strict_flags,
+                                                filename, stdout, stderr)
+            else:
+                parsed = parse_ini_config_file(options, set_strict_flags,
+                                               filename, stdout, stderr)
+            if parsed:
+                break
+
+
+def parse_toml_config_file(options: Options, set_strict_flags: Callable[[], None],
+                           filename: str,
+                           stdout: Optional[TextIO] = None,
+                           stderr: Optional[TextIO] = None) -> bool:
     stderr = stderr or sys.stderr
 
-    if filename is not None:
-        config_files = (filename,)  # type: Tuple[str, ...]
+    # Load the toml config file.
+    try:
+        table = toml.load(filename)  # type: MutableMapping[str, Any]
+    except (TypeError, toml.TomlDecodeError, IOError) as err:
+        print("%s: %s" % (filename, err), file=stderr)
+        return False
     else:
-        config_files = tuple(map(os.path.expanduser, defaults.CONFIG_FILES))
+        options.config_file = filename
 
-    parser = configparser.RawConfigParser()
+    if 'mypy' not in table:
+        print("%s: No [mypy] table in config file" % filename, file=stderr)
+        return False
 
-    for config_file in config_files:
-        if not os.path.exists(config_file):
-            continue
-        try:
-            parser.read(config_file)
-        except configparser.Error as err:
-            print("%s: %s" % (config_file, err), file=stderr)
+    # Handle the mypy table.
+    for key, value in table['mypy'].items():
+
+        # Is an option.
+        if key != 'overrides':
+
+            # Is a report directory.
+            if key.endswith('_report'):
+                report_type = key[:-7].replace('_', '-')
+                if report_type in defaults.REPORTER_NAMES:
+                    options.report_dirs[report_type] = table['mypy'][key]
+                else:
+                    print("%s: Unrecognized report type: %s" %
+                          (filename, key),
+                          file=stderr)
+            elif key == 'strict':
+                set_strict_flags()
+            else:
+                if key in toml_type_converters:
+                    value = toml_type_converters[key](value)  # type: ignore
+                setattr(options, key, value)
+
+        # Read the per-module override sub-tables.
         else:
-            file_read = config_file
-            options.config_file = file_read
-            break
+            for glob, override in value.items():
+                if (any(c in glob for c in '?[]!') or
+                        any('*' in x and x != '*' for x in glob.split('.'))):
+                    print("%s: Patterns must be fully-qualified module names, optionally "
+                          "with '*' in some components (e.g spam.*.eggs.*)"
+                          % filename, file=stderr)
+
+                values = {}
+                for subkey, subvalue in override.items():
+                    if subkey.endswith('_report'):
+                        print("Per-module override [%s] should not specify reports (%s)" %
+                              (glob, subkey), file=stderr)
+                        continue
+                    elif subkey not in PER_MODULE_OPTIONS:
+                        print("Per-module tables [%s] should only specify per-module flags (%s)" %
+                              (key, subkey), file=stderr)
+                        continue
+
+                    if subkey in toml_type_converters:
+                        subvalue = toml_type_converters[subkey](subvalue)  # type: ignore
+                    values[subkey] = subvalue
+
+                options.per_module_options[glob] = values
+    return True
+
+
+def parse_ini_config_file(options: Options, set_strict_flags: Callable[[], None],
+                          filename: str,
+                          stdout: Optional[TextIO] = None,
+                          stderr: Optional[TextIO] = None) -> bool:
+    stderr = stderr or sys.stderr
+    parser = configparser.RawConfigParser()
+    retv = False
+
+    try:
+        parser.read(filename)
+    except configparser.Error as err:
+        print("%s: %s" % (filename, err), file=stderr)
+        return retv
     else:
-        return
+        options.config_file = filename
 
     if 'mypy' not in parser:
-        if filename or file_read not in defaults.SHARED_CONFIG_FILES:
-            print("%s: No [mypy] section in config file" % file_read, file=stderr)
+        if filename not in defaults.SHARED_CONFIG_FILES:
+            print("%s: No [mypy] section in config file" % filename, file=stderr)
     else:
+        retv = True
         section = parser['mypy']
-        prefix = '%s: [%s]: ' % (file_read, 'mypy')
-        updates, report_dirs = parse_section(prefix, options, set_strict_flags, section, stderr)
+        prefix = '%s: [%s]: ' % (filename, 'mypy')
+        updates, report_dirs = parse_ini_section(
+            prefix, options, set_strict_flags, section, stderr)
         for k, v in updates.items():
             setattr(options, k, v)
         options.report_dirs.update(report_dirs)
 
     for name, section in parser.items():
         if name.startswith('mypy-'):
-            prefix = '%s: [%s]: ' % (file_read, name)
-            updates, report_dirs = parse_section(
+            retv = True
+            prefix = '%s: [%s]: ' % (filename, name)
+            updates, report_dirs = parse_ini_section(
                 prefix, options, set_strict_flags, section, stderr)
             if report_dirs:
                 print("%sPer-module sections should not specify reports (%s)" %
@@ -162,13 +269,14 @@ def parse_config_file(options: Options, set_strict_flags: Callable[[], None],
                           file=stderr)
                 else:
                     options.per_module_options[glob] = updates
+    return retv
 
 
-def parse_section(prefix: str, template: Options,
-                  set_strict_flags: Callable[[], None],
-                  section: Mapping[str, str],
-                  stderr: TextIO = sys.stderr
-                  ) -> Tuple[Dict[str, object], Dict[str, str]]:
+def parse_ini_section(prefix: str, template: Options,
+                      set_strict_flags: Callable[[], None],
+                      section: Mapping[str, str],
+                      stderr: TextIO = sys.stderr
+                      ) -> Tuple[Dict[str, object], Dict[str, str]]:
     """Parse one section of a config file.
 
     Returns a dict of option values encountered, and a dict of report directories.
@@ -178,8 +286,8 @@ def parse_section(prefix: str, template: Options,
     for key in section:
         invert = False
         options_key = key
-        if key in config_types:
-            ct = config_types[key]
+        if key in ini_type_converters:
+            ct = ini_type_converters[key]
         else:
             dv = None
             # We have to keep new_semantic_analyzer in Options
@@ -337,7 +445,7 @@ def set_strict_flags() -> None:
             nonlocal strict_found
             strict_found = True
 
-        new_sections, reports = parse_section(
+        new_sections, reports = parse_ini_section(
             '', template, set_strict_flags, parser['dummy'], stderr=stderr)
         errors.extend((lineno, x) for x in stderr.getvalue().strip().split('\n') if x)
         if reports:
diff --git a/mypy/defaults.py b/mypy/defaults.py
@@ -6,13 +6,18 @@
 PYTHON3_VERSION = (3, 6)  # type: Final
 PYTHON3_VERSION_MIN = (3, 4)  # type: Final
 CACHE_DIR = '.mypy_cache'  # type: Final
-CONFIG_FILE = 'mypy.ini'  # type: Final
+INI_CONFIG_FILE = 'mypy.ini'  # type: Final
 SHARED_CONFIG_FILES = ['setup.cfg', ]  # type: Final
 USER_CONFIG_FILES = ['~/.config/mypy/config', '~/.mypy.ini', ]  # type: Final
 if os.environ.get('XDG_CONFIG_HOME'):
     USER_CONFIG_FILES.insert(0, os.path.join(os.environ['XDG_CONFIG_HOME'], 'mypy/config'))
+TOML_CONFIG_FILE = 'pyproject.toml'  # type: Final
+CONFIG_FILES = (
+    [INI_CONFIG_FILE, TOML_CONFIG_FILE]
+    + SHARED_CONFIG_FILES
+    + USER_CONFIG_FILES
+)  # type: Final
 
-CONFIG_FILES = [CONFIG_FILE, ] + SHARED_CONFIG_FILES + USER_CONFIG_FILES  # type: Final
 
 # This must include all reporters defined in mypy.report. This is defined here
 # to make reporter names available without importing mypy.report -- this speeds
diff --git a/mypy/test/testfinegrained.py b/mypy/test/testfinegrained.py
@@ -192,7 +192,7 @@ def get_options(self,
             options.follow_imports = 'error'
 
         for name, _ in testcase.files:
-            if 'mypy.ini' in name:
+            if os.path.basename(name) in ['mypy.ini', 'pyproject.toml']:
                 parse_config_file(options, lambda: None, name)
                 break
 
diff --git a/setup.py b/setup.py
@@ -194,6 +194,7 @@ def run(self):
       install_requires=['typed_ast >= 1.4.0, < 1.5.0',
                         'typing_extensions>=3.7.4',
                         'mypy_extensions >= 0.4.3, < 0.5.0',
+                        'toml >= 0.10.0, < 0.11.0',
                         ],
       # Same here.
       extras_require={'dmypy': 'psutil >= 4.0'},
diff --git a/test-data/unit/cmdline.test b/test-data/unit/cmdline.test
@@ -170,23 +170,34 @@ def f():
     except ZeroDivisionError, err:
         print err
 
-[case testNoConfigFile]
-# cmd: mypy main.py --config-file=
-[file mypy.ini]
+[case testWarnUnusedIgnores]
+# cmd: mypy main.py --config-file=pyproject.toml
+[file pyproject.toml]
 \[mypy]
-warn_unused_ignores = True
+warn_unused_ignores = true
+[file main.py]
+# type: ignore
+[out]
+main.py:1: error: unused 'type: ignore' comment
+
+[case testMissingMypySection]
+# cmd: mypy main.py --config-file=pyproject.toml
+[file pyproject.toml]
 [file main.py]
 # type: ignore
+[out]
+pyproject.toml: No [mypy] table in config file
+== Return code: 0
 
 [case testPerFileConfigSection]
 # cmd: mypy x.py y.py z.py
-[file mypy.ini]
+[file pyproject.toml]
 \[mypy]
-disallow_untyped_defs = True
-\[mypy-y]
-disallow_untyped_defs = False
-\[mypy-z]
-disallow_untyped_calls = True
+disallow_untyped_defs = true
+\[mypy.overrides.y]
+disallow_untyped_defs = false
+\[mypy.overrides.z]
+disallow_untyped_calls = true
 [file x.py]
 def f(a):
     pass
