[stubgenc] fixes and typos (#9877)

* [stubdoc] fix typo

* [stubgenc] Add Callable to _DEFAULT_TYPING_IMPORTS

* [stubgenc] Fix parsing signatures with empty argument list

* [stubgenc] Replace NoneType with None

* [stubgenc] Skip pybind11-specific internal attributes

* [stubgenc] Test stub generation on pybind11-mypy-demo project

Author: sizmailov

.github/workflows/test_stubgenc.yml

@@ -0,0 +1,32 @@
+name: Test stubgenc on pybind11-mypy-demo
+
+on:
+  push:
+    branches: [master]
+    tags: ['*']
+  pull_request:
+    paths:
+    - 'misc/test-stubgenc.sh'
+    - 'mypy/stubgenc.py'
+    - 'mypy/stubdoc.py'
+    - 'test-data/stubgen/**'
+
+jobs:
+  stubgenc:
+    # Check stub file generation for a small pybind11 project
+    # (full text match is required to pass)
+    runs-on: ubuntu-latest
+    steps:
+
+    - uses: actions/checkout@v2
+
+    - name: initialize submodules
+      run: git submodule update --init
+
+    - name: Setup 🐍 3.8
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.8
+
+    - name: Test stubgenc
+      run: misc/test-stubgenc.sh

misc/test-stubgenc.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+# This script is expected to be run from root of the mypy repo
+
+# Install dependencies, demo project and mypy
+python -m pip install -r test-requirements.txt
+python -m pip install pybind11-mypy-demo==0.0.1
+python -m pip install .
+
+# Remove expected stubs and generate new inplace
+rm -rf test-data/stubgen/pybind11_mypy_demo
+stubgen -p pybind11_mypy_demo -o test-data/stubgen/
+
+# Compare generated stubs to expected ones
+git diff --exit-code test-data/stubgen/pybind11_mypy_demo

mypy/stubdoc.py

@@ -140,7 +140,8 @@ def add_token(self, token: tokenize.TokenInfo) -> None:
                 self.state.pop()
             elif self.state[-1] == STATE_ARGUMENT_LIST:
                 self.arg_name = self.accumulator
-                if not _ARG_NAME_RE.match(self.arg_name):
+                if not (token.string == ')' and self.accumulator.strip() == '') \
+                        and not _ARG_NAME_RE.match(self.arg_name):
                     # Invalid argument name.
                     self.reset()
                     return
@@ -235,7 +236,7 @@ def is_unique_args(sig: FunctionSig) -> bool:
         """return true if function argument names are unique"""
         return len(sig.args) == len(set((arg.name for arg in sig.args)))
 
-    # Return only signatures that have unique argument names. Mypy fails on non-uniqnue arg names.
+    # Return only signatures that have unique argument names. Mypy fails on non-unique arg names.
     return [sig for sig in sigs if is_unique_args(sig)]
 
 

mypy/stubgenc.py

@@ -17,10 +17,10 @@
     infer_arg_sig_from_anon_docstring, infer_ret_type_sig_from_anon_docstring, FunctionSig
 )
 
-
 # Members of the typing module to consider for importing by default.
 _DEFAULT_TYPING_IMPORTS = (
     'Any',
+    'Callable',
     'Dict',
     'Iterable',
     'Iterator',
@@ -231,6 +231,8 @@ def strip_or_import(typ: str, module: ModuleType, imports: List[str]) -> str:
             stripped_type = typ[len('builtins') + 1:]
         else:
             imports.append('import %s' % (arg_module,))
+    if stripped_type == 'NoneType':
+        stripped_type = 'None'
     return stripped_type
 
 
@@ -365,14 +367,20 @@ def method_name_sort_key(name: str) -> Tuple[int, str]:
     return 1, name
 
 
+def is_pybind_skipped_attribute(attr: str) -> bool:
+    return attr.startswith("__pybind11_module_local_")
+
+
 def is_skipped_attribute(attr: str) -> bool:
-    return attr in ('__getattribute__',
-                    '__str__',
-                    '__repr__',
-                    '__doc__',
-                    '__dict__',
-                    '__module__',
-                    '__weakref__')  # For pickling
+    return (attr in ('__getattribute__',
+                     '__str__',
+                     '__repr__',
+                     '__doc__',
+                     '__dict__',
+                     '__module__',
+                     '__weakref__')  # For pickling
+            or is_pybind_skipped_attribute(attr)
+            )
 
 
 def infer_method_sig(name: str) -> List[ArgSig]:

test-data/stubgen/pybind11_mypy_demo/__init__.pyi

@@ -0,0 +1,48 @@
+from typing import Any
+
+from typing import overload
+PI: float
+
+def answer() -> int: ...
+def midpoint(left: float, right: float) -> float: ...
+def sum(arg0: int, arg1: int) -> int: ...
+def weighted_midpoint(left: float, right: float, alpha: float = ...) -> float: ...
+
+class Point:
+    AngleUnit: Any = ...
+    LengthUnit: Any = ...
+    origin: Any = ...
+    @overload
+    def __init__(self) -> None: ...
+    @overload
+    def __init__(self, x: float, y: float) -> None: ...
+    @overload
+    def __init__(*args, **kwargs) -> Any: ...
+    @overload
+    def distance_to(self, x: float, y: float) -> float: ...
+    @overload
+    def distance_to(self, other: Point) -> float: ...
+    @overload
+    def distance_to(*args, **kwargs) -> Any: ...
+    @property
+    def angle_unit(self) -> pybind11_mypy_demo.basics.Point.AngleUnit: ...
+    @angle_unit.setter
+    def angle_unit(self, val: pybind11_mypy_demo.basics.Point.AngleUnit) -> None: ...
+    @property
+    def length(self) -> float: ...
+    @property
+    def length_unit(self) -> pybind11_mypy_demo.basics.Point.LengthUnit: ...
+    @length_unit.setter
+    def length_unit(self, val: pybind11_mypy_demo.basics.Point.LengthUnit) -> None: ...
+    @property
+    def x(self) -> float: ...
+    @x.setter
+    def x(self, val: float) -> None: ...
+    @property
+    def x_axis(self) -> pybind11_mypy_demo.basics.Point: ...
+    @property
+    def y(self) -> float: ...
+    @y.setter
+    def y(self, val: float) -> None: ...
+    @property
+    def y_axis(self) -> pybind11_mypy_demo.basics.Point: ...