Advancing support for pyproject.toml

Author: pslacerda

mypy/config_parser.py

@@ -6,6 +6,7 @@
 import re
 import sys
 
+import toml  # type: ignore
 from typing import Any, Dict, List, Mapping, Optional, Tuple, TextIO
 from typing_extensions import Final
 
@@ -48,11 +49,22 @@ def split_and_match_files(paths: str) -> List[str]:
 
     Where a path/glob matches no file, we still include the raw path in the resulting list.
 
+    Returns a list of file paths
+    """
+
+    return match_files(paths.split(','))
+
+
+def match_files(paths: List[str]) -> List[str]:
+    """Take list of files/directories (with support for globbing through the glob library).
+
+    Where a path/glob matches no file, we still include the raw path in the resulting list.
+
     Returns a list of file paths
     """
     expanded_paths = []
 
-    for path in paths.split(','):
+    for path in paths:
         path = expand_path(path.strip())
         globbed_files = fileglob.glob(path, recursive=True)
         if globbed_files:
@@ -67,7 +79,7 @@ def split_and_match_files(paths: str) -> List[str]:
 # sufficient, and we don't have to do anything here.  This table
 # exists to specify types for values initialized to None or container
 # types.
-config_types = {
+ini_config_types = {
     'python_version': parse_version,
     'strict_optional_whitelist': lambda s: s.split(),
     'custom_typing_module': str,
@@ -88,6 +100,16 @@ def split_and_match_files(paths: str) -> List[str]:
 }  # type: Final
 
 
+toml_type_conversors = {
+    'python_version': parse_version,
+    'custom_typeshed_dir': expand_path,
+    'mypy_path': lambda l: [expand_path(p) for p in l],
+    'files': match_files,
+    'cache_dir': expand_path,
+    'python_executable': expand_path,
+}
+
+
 def parse_config_file(options: Options, filename: Optional[str],
                       stdout: Optional[TextIO] = None,
                       stderr: Optional[TextIO] = None) -> None:
@@ -97,13 +119,90 @@ def parse_config_file(options: Options, filename: Optional[str],
 
     If filename is None, fall back to default config files.
     """
+    if filename is None or os.path.splitext(filename)[1] == '.toml':
+        parse_toml_config_file(options, filename, stderr)
+    else:
+        parse_ini_config_file(options, filename, stdout, stderr)
+
+
+def parse_toml_config_file(options: Options, filename: Optional[str],
+                           stdout: Optional[TextIO] = None,
+                           stderr: Optional[TextIO] = None) -> None:
+    stderr = stderr or sys.stderr
+
+    # Load the toml config file,
+    config_file = filename or defaults.TOML_CONFIG_FILE
+    if not os.path.exists(config_file):
+        return
+    try:
+        table = toml.load(config_file)  # type: Dict[str, Any]
+    except (TypeError, toml.TomlDecodeError, IOError) as err:
+        print("%s: %s" % (config_file, err), file=stderr)
+        return
+    else:
+        options.config_file = config_file
+
+    if 'mypy' not in table:
+        print("%s: No [mypy] table in config file" % config_file, file=stderr)
+        return
+
+    # Handle the mypy table.
+    for key, value in table['mypy'].items():
+
+        # Is an option.
+        if not isinstance(value, dict):
+
+            # Is a report directory.
+            if key.endswith('_report'):
+                report_type = key[:-7].replace('_', '-')
+                if report_type in defaults.REPORTER_NAMES:
+                    options.report_dirs[report_type] = table['mypy'][key]
+                else:
+                    print("%s: Unrecognized report type: %s" %
+                          (config_file, key),
+                          file=stderr)
+            else:
+                if key in toml_type_conversors:
+                    value = toml_type_conversors[key](value)
+                setattr(options, key, value)
+
+        # Is a subtable. Should be a package/module glob.
+        else:
+            glob = key
+            if (any(c in glob for c in '?[]!') or
+                    any('*' in x and x != '*' for x in glob.split('.'))):
+                print("%s: Patterns must be fully-qualified module names, optionally "
+                      "with '*' in some components (e.g spam.*.eggs.*)"
+                      % config_file, file=stderr)
+
+            values = {}
+            for subkey, value in table['mypy'][key].items():
+                if subkey.endswith('_report'):
+                    print("Per-module tables [%s] should not specify reports (%s)" %
+                          (key, subkey), file=stderr)
+                    continue
+                elif subkey not in PER_MODULE_OPTIONS:
+                    print("Per-module tables [%s] should only specify per-module flags (%s)" %
+                          (key, subkey), file=stderr)
+                    continue
+
+                if subkey in toml_type_conversors:
+                    value = toml_type_conversors[subkey](value)
+                values[subkey] = value
+
+            options.per_module_options[glob] = values
+
+
+def parse_ini_config_file(options: Options, filename: Optional[str],
+                          stdout: Optional[TextIO] = None,
+                          stderr: Optional[TextIO] = None) -> None:
     stdout = stdout or sys.stdout
     stderr = stderr or sys.stderr
 
     if filename is not None:
         config_files = (filename,)  # type: Tuple[str, ...]
     else:
-        config_files = tuple(map(os.path.expanduser, defaults.CONFIG_FILES))
+        config_files = tuple(map(os.path.expanduser, defaults.INI_CONFIG_FILES))
 
     parser = configparser.RawConfigParser()
 
@@ -175,8 +274,8 @@ def parse_section(prefix: str, template: Options,
     for key in section:
         invert = False
         options_key = key
-        if key in config_types:
-            ct = config_types[key]
+        if key in ini_config_types:
+            ct = ini_config_types[key]
         else:
             dv = None
             # We have to keep new_semantic_analyzer in Options

mypy/defaults.py

@@ -6,13 +6,13 @@
 PYTHON3_VERSION = (3, 6)  # type: Final
 PYTHON3_VERSION_MIN = (3, 4)  # type: Final
 CACHE_DIR = '.mypy_cache'  # type: Final
-CONFIG_FILE = 'mypy.ini'  # type: Final
+INI_CONFIG_FILE = 'mypy.ini'  # type: Final
 SHARED_CONFIG_FILES = ['setup.cfg', ]  # type: Final
 USER_CONFIG_FILES = ['~/.config/mypy/config', '~/.mypy.ini', ]  # type: Final
 if os.environ.get('XDG_CONFIG_HOME'):
     USER_CONFIG_FILES.insert(0, os.path.join(os.environ['XDG_CONFIG_HOME'], 'mypy/config'))
-
-CONFIG_FILES = [CONFIG_FILE, ] + SHARED_CONFIG_FILES + USER_CONFIG_FILES  # type: Final
+TOML_CONFIG_FILE = 'pyproject.toml'
+INI_CONFIG_FILES = [INI_CONFIG_FILE, ] + SHARED_CONFIG_FILES + USER_CONFIG_FILES
 
 # This must include all reporters defined in mypy.report. This is defined here
 # to make reporter names available without importing mypy.report -- this speeds

mypy/main.py

@@ -437,7 +437,7 @@ def add_invertible_flag(flag: str,
     config_group.add_argument(
         '--config-file',
         help="Configuration file, must have a [mypy] section "
-             "(defaults to {})".format(', '.join(defaults.CONFIG_FILES)))
+             "(defaults to {})".format(', '.join(defaults.INI_CONFIG_FILES)))
     add_invertible_flag('--warn-unused-configs', default=False, strict_flag=True,
                         help="Warn about unused '[mypy-<pattern>]' config sections",
                         group=config_group)

mypy/test/testfinegrained.py

@@ -192,7 +192,7 @@ def get_options(self,
             options.follow_imports = 'error'
 
         for name, _ in testcase.files:
-            if 'mypy.ini' in name:
+            if os.path.basename(name) in ['mypy.ini', 'pyproject.toml']:
                 parse_config_file(options, name)
                 break
 

setup.py

@@ -190,6 +190,7 @@ def run(self):
       install_requires=['typed_ast >= 1.4.0, < 1.5.0',
                         'typing_extensions>=3.7.4',
                         'mypy_extensions >= 0.4.3, < 0.5.0',
+                        'toml >= 0.10.0, < 0.11.0',
                         ],
       # Same here.
       extras_require={'dmypy': 'psutil >= 4.0'},

test-data/unit/check-custom-plugin.test

@@ -20,28 +20,28 @@ reveal_type(f())  # N: Revealed type is 'builtins.int'
 plugins=fnplugin
 
 [case testFunctionPluginFullnameIsNotNone]
-# flags: --config-file tmp/mypy.ini
+# flags: --config-file tmp/mypy.toml
 from typing import Callable, TypeVar
 f: Callable[[], None]
 T = TypeVar('T')
 def g(x: T) -> T: return x  # This strips out the name of a callable
 g(f)()
-[file mypy.ini]
-\[mypy]
-plugins=<ROOT>/test-data/unit/plugins/fnplugin.py
+[file mypy.toml]
+\[tool.mypy]
+plugins="<ROOT>/test-data/unit/plugins/fnplugin.py"
 
 [case testTwoPlugins]
-# flags: --config-file tmp/mypy.ini
+# flags: --config-file tmp/mypy.toml
 def f(): ...
 def g(): ...
 def h(): ...
-reveal_type(f())  # N: Revealed type is 'builtins.int'
-reveal_type(g())  # N: Revealed type is 'builtins.str'
-reveal_type(h())  # N: Revealed type is 'Any'
-[file mypy.ini]
-\[mypy]
-plugins=<ROOT>/test-data/unit/plugins/fnplugin.py,
-  <ROOT>/test-data/unit/plugins/plugin2.py
+reveal_type(f())  # E: Revealed type is 'builtins.int'
+reveal_type(g())  # E: Revealed type is 'builtins.str'
+reveal_type(h())  # E: Revealed type is 'Any'
+[file mypy.toml]
+\[tool.mypy]
+plugins=["<ROOT>/test-data/unit/plugins/fnplugin.py",
+  "<ROOT>/test-data/unit/plugins/plugin2.py"]
 
 [case testTwoPluginsMixedType]
 # flags: --config-file tmp/mypy.ini

test-data/unit/check-flags.test

@@ -479,7 +479,7 @@ disallow_incomplete_defs = True
 
 
 [case testPerFileStrictOptionalBasic]
-# flags: --config-file tmp/mypy.ini
+# flags: --config-file tmp/pyproject.toml
 import standard, optional
 
 [file standard.py]
@@ -491,12 +491,11 @@ x = 0
 if int():
     x = None  # E: Incompatible types in assignment (expression has type "None", variable has type "int")
 
-[file mypy.ini]
-\[mypy]
-strict_optional = False
-\[mypy-optional]
-strict_optional = True
-
+[file pyproject.toml]
+\[tool.mypy]
+strict_optional = false
+\[tool.mypy-optional]
+strict_optional = true
 
 [case testPerFileStrictOptionalBasicImportStandard]
 # flags: --config-file tmp/mypy.ini

test-data/unit/fine-grained.test

@@ -7768,10 +7768,10 @@ def f() -> str: return '0'
 ==
 
 [case testRefreshIgnoreErrors1]
-[file mypy.ini]
-\[mypy]
-\[mypy-b]
-ignore_errors = True
+[file pyproject.toml]
+\[tool.mypy]
+\[tool.mypy-b]
+ignore_errors = true
 [file a.py]
 y = '1'
 [file a.py.2]