Add movable cast support to type casters

This commit allows type_casters to allow their local values to be moved
away, rather than copied, when the type caster instance itself is an rvalue.

This only applies (automatically) to type casters using
PYBIND11_TYPE_CASTER; the generic type type casters don't own their own
pointer, and various value casters (e.g. std::string, std::pair,
arithmetic types) already cast to an rvalue (i.e. they return by value).

This updates various calling code to attempt to get a movable value
whenever the value is itself coming from a type caster about to be
destroyed: for example, when constructing an std::pair or various stl.h
containers.  For types that don't support value moving, the cast_op
falls back to an lvalue cast.

There wasn't an obvious place to add the tests, so I added them to
test_copy_move_policies, but also renamed it to drop the _policies as it
now tests more than just policies.

Author: jagerman

include/pybind11/cast.h

@@ -374,11 +374,31 @@ class type_caster_generic {
     object temp;
 };
 
-/* Determine suitable casting operator */
+/** Determine suitable casting operator for pointer-or-lvalue-casting type casters.  The type caster
+ * needs to provide `operator T*()` and `operator T&()` operators.
+ *
+ * If the type supports moving the value away via an `operator T&&() &&` method, it should use
+ * `movable_cast_op_type` instead.
+ */
 template <typename T>
-using cast_op_type = typename std::conditional<std::is_pointer<typename std::remove_reference<T>::type>::value,
-    typename std::add_pointer<intrinsic_t<T>>::type,
-    typename std::add_lvalue_reference<intrinsic_t<T>>::type>::type;
+using cast_op_type =
+    conditional_t<std::is_pointer<typename std::remove_reference<T>::type>::value,
+        typename std::add_pointer<intrinsic_t<T>>::type,
+        typename std::add_lvalue_reference<intrinsic_t<T>>::type>;
+
+/** Determine suitable casting operator for a type caster with a movable value.  Such a type caster
+ * needs to provide `operator T*()`, `operator T&()`, and `operator T&&() &&`.  The latter will be
+ * called in appropriate contexts where the value can be moved rather than copied.
+ *
+ * These operator are automatically provided when using the PYBIND11_TYPE_CASTER macro.
+ */
+template <typename T>
+using movable_cast_op_type =
+    conditional_t<std::is_pointer<typename std::remove_reference<T>::type>::value,
+        typename std::add_pointer<intrinsic_t<T>>::type,
+    conditional_t<std::is_rvalue_reference<T>::value,
+        typename std::add_rvalue_reference<intrinsic_t<T>>::type,
+        typename std::add_lvalue_reference<intrinsic_t<T>>::type>>;
 
 // std::is_copy_constructible isn't quite enough: it lets std::vector<T> (and similar) through when
 // T is non-copyable, but code containing such a copy constructor fails to actually compile.
@@ -455,7 +475,7 @@ template <typename type> class type_caster_base : public type_caster_generic {
             nullptr, nullptr, holder);
     }
 
-    template <typename T> using cast_op_type = pybind11::detail::cast_op_type<T>;
+    template <typename T> using cast_op_type = cast_op_type<T>;
 
     operator itype*() { return (type *) value; }
     operator itype&() { if (!value) throw reference_cast_error(); return *((itype *) value); }
@@ -491,8 +511,10 @@ template <typename type> using make_caster = type_caster<intrinsic_t<type>>;
 template <typename T> typename make_caster<T>::template cast_op_type<T> cast_op(make_caster<T> &caster) {
     return caster.operator typename make_caster<T>::template cast_op_type<T>();
 }
-template <typename T> typename make_caster<T>::template cast_op_type<T> cast_op(make_caster<T> &&caster) {
-    return cast_op<T>(caster);
+template <typename T> typename make_caster<T>::template cast_op_type<typename std::add_rvalue_reference<T>::type>
+cast_op(make_caster<T> &&caster) {
+    return std::move(caster).operator
+        typename make_caster<T>::template cast_op_type<typename std::add_rvalue_reference<T>::type>();
 }
 
 template <typename type> class type_caster<std::reference_wrapper<type>> : public type_caster_base<type> {
@@ -515,7 +537,8 @@ template <typename type> class type_caster<std::reference_wrapper<type>> : publi
         } \
         operator type*() { return &value; } \
         operator type&() { return value; } \
-        template <typename _T> using cast_op_type = pybind11::detail::cast_op_type<_T>
+        operator type&&() && { return std::move(value); } \
+        template <typename _T> using cast_op_type = pybind11::detail::movable_cast_op_type<_T>
 
 
 template <typename CharT> using is_std_char_type = any_of<
@@ -885,9 +908,8 @@ template <typename T1, typename T2> class type_caster<std::pair<T1, T2>> {
 
     template <typename T> using cast_op_type = type;
 
-    operator type() {
-        return type(cast_op<T1>(first), cast_op<T2>(second));
-    }
+    operator type() & { return type(cast_op<T1>(first), cast_op<T2>(second)); }
+    operator type() && { return type(cast_op<T1>(std::move(first)), cast_op<T2>(std::move(second))); }
 protected:
     make_caster<T1> first;
     make_caster<T2> second;
@@ -918,17 +940,21 @@ template <typename... Tuple> class type_caster<std::tuple<Tuple...>> {
 
     template <typename T> using cast_op_type = type;
 
-    operator type() { return implicit_cast(indices{}); }
+    operator type() & { return implicit_cast(indices{}); }
+    operator type() && { return std::move(*this).implicit_cast(indices{}); }
 
 protected:
     template <size_t... Is>
-    type implicit_cast(index_sequence<Is...>) { return type(cast_op<Tuple>(std::get<Is>(value))...); }
+    type implicit_cast(index_sequence<Is...>) & { return type(cast_op<Tuple>(std::get<Is>(subcasters))...); }
+    template <size_t... Is>
+    type implicit_cast(index_sequence<Is...>) && { return type(cast_op<Tuple>(std::move(std::get<Is>(subcasters)))...); }
+
 
     static constexpr bool load_impl(const sequence &, bool, index_sequence<>) { return true; }
 
     template <size_t... Is>
     bool load_impl(const sequence &seq, bool convert, index_sequence<Is...>) {
-        for (bool r : {std::get<Is>(value).load(seq[Is], convert)...})
+        for (bool r : {std::get<Is>(subcasters).load(seq[Is], convert)...})
             if (!r)
                 return false;
         return true;
@@ -953,7 +979,7 @@ template <typename... Tuple> class type_caster<std::tuple<Tuple...>> {
         return result.release();
     }
 
-    std::tuple<make_caster<Tuple>...> value;
+    std::tuple<make_caster<Tuple>...> subcasters;
 };
 
 /// Helper class which abstracts away certain actions. Users can provide specializations for
@@ -1458,13 +1484,13 @@ class argument_loader {
     }
 
     template <typename Return, typename Guard, typename Func>
-    enable_if_t<!std::is_void<Return>::value, Return> call(Func &&f) {
-        return call_impl<Return>(std::forward<Func>(f), indices{}, Guard{});
+    enable_if_t<!std::is_void<Return>::value, Return> call(Func &&f) && {
+        return std::move(*this).template call_impl<Return>(std::forward<Func>(f), indices{}, Guard{});
     }
 
     template <typename Return, typename Guard, typename Func>
-    enable_if_t<std::is_void<Return>::value, void_type> call(Func &&f) {
-        call_impl<Return>(std::forward<Func>(f), indices{}, Guard{});
+    enable_if_t<std::is_void<Return>::value, void_type> call(Func &&f) && {
+        std::move(*this).template call_impl<Return>(std::forward<Func>(f), indices{}, Guard{});
         return void_type();
     }
 
@@ -1474,18 +1500,18 @@ class argument_loader {
 
     template <size_t... Is>
     bool load_impl_sequence(function_call &call, index_sequence<Is...>) {
-        for (bool r : {std::get<Is>(value).load(call.args[Is], call.args_convert[Is])...})
+        for (bool r : {std::get<Is>(argcasters).load(call.args[Is], call.args_convert[Is])...})
             if (!r)
                 return false;
         return true;
     }
 
     template <typename Return, typename Func, size_t... Is, typename Guard>
     Return call_impl(Func &&f, index_sequence<Is...>, Guard &&) {
-        return std::forward<Func>(f)(cast_op<Args>(std::get<Is>(value))...);
+        return std::forward<Func>(f)(cast_op<Args>(std::move(std::get<Is>(argcasters)))...);
     }
 
-    std::tuple<make_caster<Args>...> value;
+    std::tuple<make_caster<Args>...> argcasters;
 };
 
 /// Helper class which collects only positional arguments for a Python function call.

include/pybind11/eigen.h

@@ -341,7 +341,8 @@ struct type_caster<Type, enable_if_t<is_eigen_dense_plain<Type>::value>> {
 
     operator Type*() { return &value; }
     operator Type&() { return value; }
-    template <typename T> using cast_op_type = cast_op_type<T>;
+    operator Type&&() && { return std::move(value); }
+    template <typename T> using cast_op_type = movable_cast_op_type<T>;
 
 private:
     Type value;

include/pybind11/pybind11.h

@@ -150,8 +150,8 @@ class cpp_function : public function {
             using Guard = detail::extract_guard_t<Extra...>;
 
             /* Perform the function call */
-            handle result = cast_out::cast(args_converter.template call<Return, Guard>(cap->f),
-                                           policy, call.parent);
+            handle result = cast_out::cast(
+                std::move(args_converter).template call<Return, Guard>(cap->f), policy, call.parent);
 
             /* Invoke call policy post-call hook */
             detail::process_attributes<Extra...>::postcall(call, result);

include/pybind11/stl.h

@@ -60,11 +60,11 @@ template <typename Type, typename Key> struct set_caster {
             return false;
         auto s = reinterpret_borrow<pybind11::set>(src);
         value.clear();
-        key_conv conv;
         for (auto entry : s) {
+            key_conv conv;
             if (!conv.load(entry, convert))
                 return false;
-            value.insert(cast_op<Key>(conv));
+            value.insert(cast_op<Key &&>(std::move(conv)));
         }
         return true;
     }
@@ -90,14 +90,14 @@ template <typename Type, typename Key, typename Value> struct map_caster {
         if (!isinstance<dict>(src))
             return false;
         auto d = reinterpret_borrow<dict>(src);
-        key_conv kconv;
-        value_conv vconv;
         value.clear();
         for (auto it : d) {
+            key_conv kconv;
+            value_conv vconv;
             if (!kconv.load(it.first.ptr(), convert) ||
                 !vconv.load(it.second.ptr(), convert))
                 return false;
-            value.emplace(cast_op<Key>(kconv), cast_op<Value>(vconv));
+            value.emplace(cast_op<Key &&>(std::move(kconv)), cast_op<Value &&>(std::move(vconv)));
         }
         return true;
     }
@@ -124,13 +124,13 @@ template <typename Type, typename Value> struct list_caster {
         if (!isinstance<sequence>(src))
             return false;
         auto s = reinterpret_borrow<sequence>(src);
-        value_conv conv;
         value.clear();
         reserve_maybe(s, &value);
         for (auto it : s) {
+            value_conv conv;
             if (!conv.load(it, convert))
                 return false;
-            value.push_back(cast_op<Value>(conv));
+            value.push_back(cast_op<Value &&>(std::move(conv)));
         }
         return true;
     }
@@ -185,12 +185,12 @@ template <typename ArrayType, typename Value, bool Resizable, size_t Size = 0> s
         auto l = reinterpret_borrow<list>(src);
         if (!require_size(l.size()))
             return false;
-        value_conv conv;
         size_t ctr = 0;
         for (auto it : l) {
+            value_conv conv;
             if (!conv.load(it, convert))
                 return false;
-            value[ctr++] = cast_op<Value>(conv);
+            value[ctr++] = cast_op<Value &&>(std::move(conv));
         }
         return true;
     }
@@ -249,7 +249,7 @@ template<typename T> struct optional_caster {
         if (!inner_caster.load(src, convert))
             return false;
 
-        value.emplace(cast_op<typename T::value_type>(inner_caster));
+        value.emplace(cast_op<typename T::value_type &&>(std::move(inner_caster)));
         return true;
     }
 

tests/CMakeLists.txt

@@ -33,7 +33,7 @@ set(PYBIND11_TEST_FILES
   test_chrono.cpp
   test_class_args.cpp
   test_constants_and_functions.cpp
-  test_copy_move_policies.cpp
+  test_copy_move.cpp
   test_docstring_options.cpp
   test_eigen.cpp
   test_enum.cpp