Fix *args/**kwargs return types. Add type hinting to py::make_tuple (#5881)

* Type hint make_tuple / fix *args/**kwargs return type

Signed-off-by: Michael Carlstrom <[email protected]>

* add back commented out panic

* ignore return std move clang

Signed-off-by: Michael Carlstrom <[email protected]>

* fix for mingmw

Signed-off-by: Michael Carlstrom <[email protected]>

* added missing case

Signed-off-by: Michael Carlstrom <[email protected]>

---------

Signed-off-by: Michael Carlstrom <[email protected]>

Author: InvincibleRMC

include/pybind11/cast.h

@@ -1468,21 +1468,24 @@ template <>
 struct handle_type_name<weakref> {
     static constexpr auto name = const_name("weakref.ReferenceType");
 };
+// args/Args/kwargs/KWArgs have name as well as typehint included
 template <>
 struct handle_type_name<args> {
-    static constexpr auto name = const_name("*args");
+    static constexpr auto name = io_name("*args", "tuple");
 };
 template <typename T>
 struct handle_type_name<Args<T>> {
-    static constexpr auto name = const_name("*args: ") + make_caster<T>::name;
+    static constexpr auto name
+        = io_name("*args: ", "tuple[") + make_caster<T>::name + io_name("", ", ...]");
 };
 template <>
 struct handle_type_name<kwargs> {
-    static constexpr auto name = const_name("**kwargs");
+    static constexpr auto name = io_name("**kwargs", "dict[str, typing.Any]");
 };
 template <typename T>
 struct handle_type_name<KWArgs<T>> {
-    static constexpr auto name = const_name("**kwargs: ") + make_caster<T>::name;
+    static constexpr auto name
+        = io_name("**kwargs: ", "dict[str, ") + make_caster<T>::name + io_name("", "]");
 };
 template <>
 struct handle_type_name<obj_attr_accessor> {
@@ -1908,13 +1911,20 @@ inline cast_error cast_error_unable_to_convert_call_arg(const std::string &name,
 }
 #endif
 
+namespace typing {
+template <typename... Types>
+class Tuple : public tuple {
+    using tuple::tuple;
+};
+} // namespace typing
+
 template <return_value_policy policy = return_value_policy::automatic_reference>
-tuple make_tuple() {
+typing::Tuple<> make_tuple() {
     return tuple(0);
 }
 
 template <return_value_policy policy = return_value_policy::automatic_reference, typename... Args>
-tuple make_tuple(Args &&...args_) {
+typing::Tuple<Args...> make_tuple(Args &&...args_) {
     constexpr size_t size = sizeof...(Args);
     std::array<object, size> args{{reinterpret_steal<object>(
         detail::make_caster<Args>::cast(std::forward<Args>(args_), policy, nullptr))...}};
@@ -1933,7 +1943,12 @@ tuple make_tuple(Args &&...args_) {
     for (auto &arg_value : args) {
         PyTuple_SET_ITEM(result.ptr(), counter++, arg_value.release().ptr());
     }
+    PYBIND11_WARNING_PUSH
+#ifdef PYBIND11_DETECTED_CLANG_WITH_MISLEADING_CALL_STD_MOVE_EXPLICITLY_WARNING
+    PYBIND11_WARNING_DISABLE_CLANG("-Wreturn-std-move")
+#endif
     return result;
+    PYBIND11_WARNING_POP
 }
 
 /// \ingroup annotations

include/pybind11/detail/init.h

@@ -501,9 +501,15 @@ template <typename Get,
           typename NewInstance,
           typename ArgState>
 struct pickle_factory<Get, Set, RetState(Self), NewInstance(ArgState)> {
-    static_assert(std::is_same<intrinsic_t<RetState>, intrinsic_t<ArgState>>::value,
-                  "The type returned by `__getstate__` must be the same "
-                  "as the argument accepted by `__setstate__`");
+    using Ret = intrinsic_t<RetState>;
+    using Arg = intrinsic_t<ArgState>;
+
+    // Subclasses are now allowed for support between type hint and generic versions of types
+    // (e.g.) typing::List <--> list
+    static_assert(std::is_same<Ret, Arg>::value || std::is_base_of<Ret, Arg>::value
+                      || std::is_base_of<Arg, Ret>::value,
+                  "The type returned by `__getstate__` must be the same or subclass of the "
+                  "argument accepted by `__setstate__`");
 
     remove_reference_t<Get> get;
     remove_reference_t<Set> set;

include/pybind11/pybind11.h

@@ -120,7 +120,8 @@ inline std::string generate_function_signature(const char *type_caster_name_fiel
         const auto c = *pc;
         if (c == '{') {
             // Write arg name for everything except *args and **kwargs.
-            is_starred = *(pc + 1) == '*';
+            // Detect {@*args...} or {@**kwargs...}
+            is_starred = *(pc + 1) == '@' && *(pc + 2) == '*';
             if (is_starred) {
                 continue;
             }

include/pybind11/typing.h

@@ -34,10 +34,7 @@ PYBIND11_NAMESPACE_BEGIN(typing)
     There is no additional enforcement of types at runtime.
 */
 
-template <typename... Types>
-class Tuple : public tuple {
-    using tuple::tuple;
-};
+// Tuple type hint defined in cast.h for use in py::make_tuple to avoid circular includes
 
 template <typename K, typename V>
 class Dict : public dict {

tests/test_factory_constructors.cpp

@@ -376,10 +376,14 @@ TEST_SUBMODULE(factory_constructors, m) {
             py::print("noisy placement new");
             return p;
         }
-        static void operator delete(void *p, size_t) {
+        static void operator delete(void *p) noexcept {
             py::print("noisy delete");
             ::operator delete(p);
         }
+        static void operator delete(void *p, size_t) {
+            py::print("noisy delete size");
+            ::operator delete(p);
+        }
         static void operator delete(void *, void *) { py::print("noisy placement delete"); }
     };
 

tests/test_kwargs_and_defaults.py

@@ -34,11 +34,12 @@ def test_function_signatures(doc):
     )
     assert doc(m.args_function) == "args_function(*args) -> tuple"
     assert (
-        doc(m.args_kwargs_function) == "args_kwargs_function(*args, **kwargs) -> tuple"
+        doc(m.args_kwargs_function)
+        == "args_kwargs_function(*args, **kwargs) -> tuple[tuple, dict[str, typing.Any]]"
     )
     assert (
         doc(m.args_kwargs_subclass_function)
-        == "args_kwargs_subclass_function(*args: str, **kwargs: str) -> tuple"
+        == "args_kwargs_subclass_function(*args: str, **kwargs: str) -> tuple[tuple[str, ...], dict[str, str]]"
     )
     assert (
         doc(m.KWClass.foo0)
@@ -138,7 +139,7 @@ def test_mixed_args_and_kwargs(msg):
         msg(excinfo.value)
         == """
         mixed_plus_args(): incompatible function arguments. The following argument types are supported:
-            1. (arg0: typing.SupportsInt | typing.SupportsIndex, arg1: typing.SupportsFloat | typing.SupportsIndex, *args) -> tuple
+            1. (arg0: typing.SupportsInt | typing.SupportsIndex, arg1: typing.SupportsFloat | typing.SupportsIndex, *args) -> tuple[int, float, tuple]
 
         Invoked with: 1
     """
@@ -149,7 +150,7 @@ def test_mixed_args_and_kwargs(msg):
         msg(excinfo.value)
         == """
         mixed_plus_args(): incompatible function arguments. The following argument types are supported:
-            1. (arg0: typing.SupportsInt | typing.SupportsIndex, arg1: typing.SupportsFloat | typing.SupportsIndex, *args) -> tuple
+            1. (arg0: typing.SupportsInt | typing.SupportsIndex, arg1: typing.SupportsFloat | typing.SupportsIndex, *args) -> tuple[int, float, tuple]
 
         Invoked with:
     """
@@ -183,7 +184,7 @@ def test_mixed_args_and_kwargs(msg):
         msg(excinfo.value)
         == """
         mixed_plus_args_kwargs_defaults(): incompatible function arguments. The following argument types are supported:
-            1. (i: typing.SupportsInt | typing.SupportsIndex = 1, j: typing.SupportsFloat | typing.SupportsIndex = 3.14159, *args, **kwargs) -> tuple
+            1. (i: typing.SupportsInt | typing.SupportsIndex = 1, j: typing.SupportsFloat | typing.SupportsIndex = 3.14159, *args, **kwargs) -> tuple[int, float, tuple, dict[str, typing.Any]]
 
         Invoked with: 1; kwargs: i=1
     """
@@ -194,7 +195,7 @@ def test_mixed_args_and_kwargs(msg):
         msg(excinfo.value)
         == """
         mixed_plus_args_kwargs_defaults(): incompatible function arguments. The following argument types are supported:
-            1. (i: typing.SupportsInt | typing.SupportsIndex = 1, j: typing.SupportsFloat | typing.SupportsIndex = 3.14159, *args, **kwargs) -> tuple
+            1. (i: typing.SupportsInt | typing.SupportsIndex = 1, j: typing.SupportsFloat | typing.SupportsIndex = 3.14159, *args, **kwargs) -> tuple[int, float, tuple, dict[str, typing.Any]]
 
         Invoked with: 1, 2; kwargs: j=1
     """
@@ -211,7 +212,7 @@ def test_mixed_args_and_kwargs(msg):
         msg(excinfo.value)
         == """
         args_kwonly(): incompatible function arguments. The following argument types are supported:
-            1. (i: typing.SupportsInt | typing.SupportsIndex, j: typing.SupportsFloat | typing.SupportsIndex, *args, z: typing.SupportsInt | typing.SupportsIndex) -> tuple
+            1. (i: typing.SupportsInt | typing.SupportsIndex, j: typing.SupportsFloat | typing.SupportsIndex, *args, z: typing.SupportsInt | typing.SupportsIndex) -> tuple[int, float, tuple, int]
 
         Invoked with: 2, 2.5, 22
     """
@@ -233,12 +234,12 @@ def test_mixed_args_and_kwargs(msg):
     )
     assert (
         m.args_kwonly_kwargs.__doc__
-        == "args_kwonly_kwargs(i: typing.SupportsInt | typing.SupportsIndex, j: typing.SupportsFloat | typing.SupportsIndex, *args, z: typing.SupportsInt | typing.SupportsIndex, **kwargs) -> tuple\n"
+        == "args_kwonly_kwargs(i: typing.SupportsInt | typing.SupportsIndex, j: typing.SupportsFloat | typing.SupportsIndex, *args, z: typing.SupportsInt | typing.SupportsIndex, **kwargs) -> tuple[int, float, tuple, int, dict[str, typing.Any]]\n"
     )
 
     assert (
         m.args_kwonly_kwargs_defaults.__doc__
-        == "args_kwonly_kwargs_defaults(i: typing.SupportsInt | typing.SupportsIndex = 1, j: typing.SupportsFloat | typing.SupportsIndex = 3.14159, *args, z: typing.SupportsInt | typing.SupportsIndex = 42, **kwargs) -> tuple\n"
+        == "args_kwonly_kwargs_defaults(i: typing.SupportsInt | typing.SupportsIndex = 1, j: typing.SupportsFloat | typing.SupportsIndex = 3.14159, *args, z: typing.SupportsInt | typing.SupportsIndex = 42, **kwargs) -> tuple[int, float, tuple, int, dict[str, typing.Any]]\n"
     )
     assert m.args_kwonly_kwargs_defaults() == (1, 3.14159, (), 42, {})
     assert m.args_kwonly_kwargs_defaults(2) == (2, 3.14159, (), 42, {})
@@ -344,7 +345,7 @@ def test_positional_only_args():
     # Mix it with args and kwargs:
     assert (
         m.args_kwonly_full_monty.__doc__
-        == "args_kwonly_full_monty(arg0: typing.SupportsInt | typing.SupportsIndex = 1, arg1: typing.SupportsInt | typing.SupportsIndex = 2, /, j: typing.SupportsFloat | typing.SupportsIndex = 3.14159, *args, z: typing.SupportsInt | typing.SupportsIndex = 42, **kwargs) -> tuple\n"
+        == "args_kwonly_full_monty(arg0: typing.SupportsInt | typing.SupportsIndex = 1, arg1: typing.SupportsInt | typing.SupportsIndex = 2, /, j: typing.SupportsFloat | typing.SupportsIndex = 3.14159, *args, z: typing.SupportsInt | typing.SupportsIndex = 42, **kwargs) -> tuple[int, int, float, tuple, int, dict[str, typing.Any]]\n"
     )
     assert m.args_kwonly_full_monty() == (1, 2, 3.14159, (), 42, {})
     assert m.args_kwonly_full_monty(8) == (8, 2, 3.14159, (), 42, {})
@@ -394,23 +395,23 @@ def test_positional_only_args():
 def test_signatures():
     assert (
         m.kw_only_all.__doc__
-        == "kw_only_all(*, i: typing.SupportsInt | typing.SupportsIndex, j: typing.SupportsInt | typing.SupportsIndex) -> tuple\n"
+        == "kw_only_all(*, i: typing.SupportsInt | typing.SupportsIndex, j: typing.SupportsInt | typing.SupportsIndex) -> tuple[int, int]\n"
     )
     assert (
         m.kw_only_mixed.__doc__
-        == "kw_only_mixed(i: typing.SupportsInt | typing.SupportsIndex, *, j: typing.SupportsInt | typing.SupportsIndex) -> tuple\n"
+        == "kw_only_mixed(i: typing.SupportsInt | typing.SupportsIndex, *, j: typing.SupportsInt | typing.SupportsIndex) -> tuple[int, int]\n"
     )
     assert (
         m.pos_only_all.__doc__
-        == "pos_only_all(i: typing.SupportsInt | typing.SupportsIndex, j: typing.SupportsInt | typing.SupportsIndex, /) -> tuple\n"
+        == "pos_only_all(i: typing.SupportsInt | typing.SupportsIndex, j: typing.SupportsInt | typing.SupportsIndex, /) -> tuple[int, int]\n"
     )
     assert (
         m.pos_only_mix.__doc__
-        == "pos_only_mix(i: typing.SupportsInt | typing.SupportsIndex, /, j: typing.SupportsInt | typing.SupportsIndex) -> tuple\n"
+        == "pos_only_mix(i: typing.SupportsInt | typing.SupportsIndex, /, j: typing.SupportsInt | typing.SupportsIndex) -> tuple[int, int]\n"
     )
     assert (
         m.pos_kw_only_mix.__doc__
-        == "pos_kw_only_mix(i: typing.SupportsInt | typing.SupportsIndex, /, j: typing.SupportsInt | typing.SupportsIndex, *, k: typing.SupportsInt | typing.SupportsIndex) -> tuple\n"
+        == "pos_kw_only_mix(i: typing.SupportsInt | typing.SupportsIndex, /, j: typing.SupportsInt | typing.SupportsIndex, *, k: typing.SupportsInt | typing.SupportsIndex) -> tuple[int, int, int]\n"
     )