Allow keyword-only arguments to follow py::args

This removes the constraint that py::args has to be last (or
second-last, with py::kwargs) and instead makes py::args imply
py::kw_only for any remaining arguments, allowing you to bind a function
that works the same way as a Python function such as:

    def f(a, *args, b):
        return a * b + sum(args)

    f(10, 1, 2, 3, b=20)  # == 206

With this change, you can bind such a function using:

    m.def("f", [](int a, py::args args, int b) { /* ... */ },
        "a"_a, "b"_a);

Or, to be more explicit about the keyword-only arguments:

    m.def("g", [](int a, py::args args, int b) { /* ... */ },
        "a"_a, py::kw_only{}, "b"_a);

(The only difference between the two is that the latter will fail at
binding time if the `kw_only{}` doesn't match the `py::args` position).

This doesn't affect backwards compatibility at all because, currently,
you can't have a py::args anywhere except the end/2nd-last.

Author: jagerman

docs/advanced/functions.rst

@@ -306,8 +306,9 @@ The class ``py::args`` derives from ``py::tuple`` and ``py::kwargs`` derives
 from ``py::dict``.
 
 You may also use just one or the other, and may combine these with other
-arguments as long as the ``py::args`` and ``py::kwargs`` arguments are the last
-arguments accepted by the function.
+arguments.  Note, however, that ``py::kwargs`` must always be the last argument
+of the function, and ``py::args`` implies that any further arguments are
+keyword-only (see :ref:`keyword_only_arguments`).
 
 Please refer to the other examples for details on how to iterate over these,
 and on how to cast their entries into C++ objects. A demonstration is also
@@ -366,6 +367,8 @@ like so:
     py::class_<MyClass>("MyClass")
         .def("myFunction", py::arg("arg") = static_cast<SomeType *>(nullptr));
 
+.. _keyword_only_arguments:
+
 Keyword-only arguments
 ======================
 
@@ -397,6 +400,15 @@ feature does *not* require Python 3 to work.
 
 .. versionadded:: 2.6
 
+As of pybind11 2.9, a ``py::args`` argument implies that any following arguments
+are keyword-only, as if ``py::kw_only()`` had been specified in the same
+relative location of the argument list as the ``py::args`` argument.  The
+``py::kw_only()`` may be included to be explicit about this, but is not
+required.  (Prior to 2.9 ``py::args`` may only occur at the end of the argument
+list, or immediately before a ``py::kwargs`` argument at the end).
+
+.. versionadded:: 2.9
+
 Positional-only arguments
 =========================
 

include/pybind11/attr.h

@@ -411,7 +411,7 @@ template <> struct process_attribute<is_new_style_constructor> : process_attribu
 
 inline void check_kw_only_arg(const arg &a, function_record *r) {
     if (r->args.size() > r->nargs_pos && (!a.name || a.name[0] == '\0'))
-        pybind11_fail("arg(): cannot specify an unnamed argument after an kw_only() annotation");
+        pybind11_fail("arg(): cannot specify an unnamed argument after a kw_only() annotation or args() argument");
 }
 
 /// Process a keyword argument attribute (*without* a default value)
@@ -461,6 +461,8 @@ template <> struct process_attribute<arg_v> : process_attribute_default<arg_v> {
 /// Process a keyword-only-arguments-follow pseudo argument
 template <> struct process_attribute<kw_only> : process_attribute_default<kw_only> {
     static void init(const kw_only &, function_record *r) {
+        if (r->has_args && r->nargs_pos != static_cast<std::uint16_t>(r->args.size()))
+            pybind11_fail("Mismatched args() and kw_only(): they must occur at the same relative argument location (or omit kw_only() entirely)");
         r->nargs_pos = static_cast<std::uint16_t>(r->args.size());
     }
 };
@@ -469,6 +471,9 @@ template <> struct process_attribute<kw_only> : process_attribute_default<kw_onl
 template <> struct process_attribute<pos_only> : process_attribute_default<pos_only> {
     static void init(const pos_only &, function_record *r) {
         r->nargs_pos_only = static_cast<std::uint16_t>(r->args.size());
+        if (r->nargs_pos_only > r->nargs_pos)
+            pybind11_fail("pos_only(): cannot follow a py::args() argument");
+            // It also can't follow a kw_only, but a static_assert in pybind11.h checks that
     }
 };
 

include/pybind11/cast.h

@@ -1163,19 +1163,17 @@ template <typename... Args>
 class argument_loader {
     using indices = make_index_sequence<sizeof...(Args)>;
 
-    template <typename Arg> using argument_is_args   = std::is_same<intrinsic_t<Arg>, args>;
-    template <typename Arg> using argument_is_kwargs = std::is_same<intrinsic_t<Arg>, kwargs>;
-    // Get args/kwargs argument positions relative to the end of the argument list:
-    static constexpr auto args_pos = constexpr_first<argument_is_args, Args...>() - (int) sizeof...(Args),
-                        kwargs_pos = constexpr_first<argument_is_kwargs, Args...>() - (int) sizeof...(Args);
+    // Get kwargs argument position, or -1 if not present:
+    static constexpr auto kwargs_pos = constexpr_last(std::is_same<intrinsic_t<Args>, kwargs>::value...);
 
-    static constexpr bool args_kwargs_are_last = kwargs_pos >= - 1 && args_pos >= kwargs_pos - 1;
-
-    static_assert(args_kwargs_are_last, "py::args/py::kwargs are only permitted as the last argument(s) of a function");
+    static_assert(kwargs_pos == -1 || kwargs_pos == (int) sizeof...(Args) - 1, "py::kwargs is only permitted as the last argument of a function");
+    static_assert(constexpr_sum(std::is_same<intrinsic_t<Args>, args>::value...) <= 1, "py::args cannot be specified more than once");
 
 public:
-    static constexpr bool has_kwargs = kwargs_pos < 0;
-    static constexpr bool has_args = args_pos < 0;
+    static constexpr bool has_kwargs = kwargs_pos != -1;
+
+    // py::args argument position; -1 if not present.
+    static constexpr int args_pos = constexpr_last(std::is_same<intrinsic_t<Args>, args>::value...);
 
     static constexpr auto arg_names = concat(type_descr(make_caster<Args>::name)...);
 
@@ -1381,8 +1379,8 @@ template <return_value_policy policy, typename... Args,
 unpacking_collector<policy> collect_arguments(Args &&...args) {
     // Following argument order rules for generalized unpacking according to PEP 448
     static_assert(
-        constexpr_last<is_positional, Args...>() < constexpr_first<is_keyword_or_ds, Args...>()
-        && constexpr_last<is_s_unpacking, Args...>() < constexpr_first<is_ds_unpacking, Args...>(),
+        constexpr_last(is_positional<Args>::value...) < constexpr_first(is_keyword_or_ds<Args>::value...)
+        && constexpr_last(is_s_unpacking<Args>::value...) < constexpr_first(is_ds_unpacking<Args>::value...),
         "Invalid function call: positional args must precede keywords and ** unpacking; "
         "* unpacking must precede ** unpacking"
     );

include/pybind11/detail/common.h

@@ -684,14 +684,13 @@ template <typename T, typename... Ts>
 constexpr int last(int i, int result, T v, Ts... vs) { return last(i + 1, v ? i : result, vs...); }
 PYBIND11_NAMESPACE_END(constexpr_impl)
 
-/// Return the index of the first type in Ts which satisfies Predicate<T>.  Returns sizeof...(Ts) if
-/// none match.
-template <template<typename> class Predicate, typename... Ts>
-constexpr int constexpr_first() { return constexpr_impl::first(0, Predicate<Ts>::value...); }
+/// Returns the index of the first true argument.  Returns sizeof...(Args) if none match.
+template <typename... Args>
+constexpr int constexpr_first(Args... args) { return constexpr_impl::first(0, args...); }
 
-/// Return the index of the last type in Ts which satisfies Predicate<T>, or -1 if none match.
-template <template<typename> class Predicate, typename... Ts>
-constexpr int constexpr_last() { return constexpr_impl::last(0, -1, Predicate<Ts>::value...); }
+/// Returns the index of the last true argument.  Returns -1 if none match.
+template <typename... Args>
+constexpr int constexpr_last(Args... args) { return constexpr_impl::last(0, -1, args...); }
 
 /// Return the Nth element from the parameter pack
 template <size_t N, typename T, typename... Ts>
@@ -706,7 +705,7 @@ struct exactly_one {
     static constexpr auto found = constexpr_sum(Predicate<Ts>::value...);
     static_assert(found <= 1, "Found more than one type matching the predicate");
 
-    static constexpr auto index = found ? constexpr_first<Predicate, Ts...>() : 0;
+    static constexpr auto index = found ? constexpr_first(Predicate<Ts>::value...) : 0;
     using type = conditional_t<found, typename pack_element<index, Ts...>::type, Default>;
 };
 template <template<typename> class P, typename Default>

include/pybind11/pybind11.h

@@ -203,7 +203,7 @@ class cpp_function : public function {
             conditional_t<std::is_void<Return>::value, void_type, Return>
         >;
 
-        static_assert(expected_num_args<Extra...>(sizeof...(Args), cast_in::has_args, cast_in::has_kwargs),
+        static_assert(expected_num_args<Extra...>(sizeof...(Args), cast_in::args_pos >= 0, cast_in::has_kwargs),
                       "The number of argument annotations does not match the number of function arguments");
 
         /* Dispatch code which converts function arguments and performs the actual function call */
@@ -238,19 +238,27 @@ class cpp_function : public function {
             return result;
         };
 
-        rec->nargs_pos = sizeof...(Args) - cast_in::has_args - cast_in::has_kwargs; // Will get reduced more if we have a kw_only
+        rec->nargs_pos = cast_in::args_pos >= 0
+            ? static_cast<std::uint16_t>(cast_in::args_pos)
+            : sizeof...(Args) - cast_in::has_kwargs; // Will get reduced more if we have a kw_only
+        if (cast_in::args_pos >= 0) rec->has_args = true;
+        if (cast_in::has_kwargs) rec->has_kwargs = true;
 
         /* Process any user-provided function attributes */
         process_attributes<Extra...>::init(extra..., rec);
 
         {
             constexpr bool has_kw_only_args = any_of<std::is_same<kw_only, Extra>...>::value,
                            has_pos_only_args = any_of<std::is_same<pos_only, Extra>...>::value,
-                           has_args = any_of<std::is_same<args, Args>...>::value,
                            has_arg_annotations = any_of<is_keyword<Extra>...>::value;
             static_assert(has_arg_annotations || !has_kw_only_args, "py::kw_only requires the use of argument annotations");
             static_assert(has_arg_annotations || !has_pos_only_args, "py::pos_only requires the use of argument annotations (for docstrings and aligning the annotations to the argument)");
-            static_assert(!(has_args && has_kw_only_args), "py::kw_only cannot be combined with a py::args argument");
+
+            static_assert(constexpr_sum(std::is_same<kw_only, Extra>::value...) <= 1, "py::kw_only may be specified only once");
+            static_assert(constexpr_sum(std::is_same<pos_only, Extra>::value...) <= 1, "py::pos_only may be specified only once");
+            constexpr auto kw_only_pos = constexpr_first(std::is_same<kw_only, Extra>::value...);
+            constexpr auto pos_only_pos = constexpr_first(std::is_same<pos_only, Extra>::value...);
+            static_assert(!(has_kw_only_args && has_pos_only_args) || pos_only_pos < kw_only_pos, "py::pos_only must come before py::kw_only");
         }
 
         /* Generate a readable signature describing the function's arguments and return value types */
@@ -261,9 +269,6 @@ class cpp_function : public function {
         // Pass on the ownership over the `unique_rec` to `initialize_generic`. `rec` stays valid.
         initialize_generic(std::move(unique_rec), signature.text, types.data(), sizeof...(Args));
 
-        if (cast_in::has_args) rec->has_args = true;
-        if (cast_in::has_kwargs) rec->has_kwargs = true;
-
         /* Stash some additional information used by an important optimization in 'functional.h' */
         using FunctionType = Return (*)(Args...);
         constexpr bool is_function_ptr =
@@ -342,15 +347,17 @@ class cpp_function : public function {
         /* Generate a proper function signature */
         std::string signature;
         size_t type_index = 0, arg_index = 0;
+        bool is_starred = false;
         for (auto *pc = text; *pc != '\0'; ++pc) {
             const auto c = *pc;
 
             if (c == '{') {
                 // Write arg name for everything except *args and **kwargs.
-                if (*(pc + 1) == '*')
+                is_starred = *(pc + 1) == '*';
+                if (is_starred)
                     continue;
                 // Separator for keyword-only arguments, placed before the kw
-                // arguments start
+                // arguments start (unless we are already putting an *args)
                 if (!rec->has_args && arg_index == rec->nargs_pos)
                     signature += "*, ";
                 if (arg_index < rec->args.size() && rec->args[arg_index].name) {
@@ -363,15 +370,16 @@ class cpp_function : public function {
                 signature += ": ";
             } else if (c == '}') {
                 // Write default value if available.
-                if (arg_index < rec->args.size() && rec->args[arg_index].descr) {
+                if (!is_starred && arg_index < rec->args.size() && rec->args[arg_index].descr) {
                     signature += " = ";
                     signature += rec->args[arg_index].descr;
                 }
                 // Separator for positional-only arguments (placed after the
                 // argument, rather than before like *
                 if (rec->nargs_pos_only > 0 && (arg_index + 1) == rec->nargs_pos_only)
                     signature += ", /";
-                arg_index++;
+                if (!is_starred)
+                    arg_index++;
             } else if (c == '%') {
                 const std::type_info *t = types[type_index++];
                 if (!t)
@@ -397,7 +405,7 @@ class cpp_function : public function {
             }
         }
 
-        if (arg_index != args || types[type_index] != nullptr)
+        if (arg_index != args - rec->has_args - rec->has_kwargs || types[type_index] != nullptr)
             pybind11_fail("Internal error while parsing type signature (2)");
 
 #if PY_MAJOR_VERSION < 3
@@ -697,6 +705,10 @@ class cpp_function : public function {
                 if (bad_arg)
                     continue; // Maybe it was meant for another overload (issue #688)
 
+                // Keep track of how many position args we copied out in case we need to come back
+                // to copy the rest into a py::args argument.
+                size_t positional_args_copied = args_copied;
+
                 // We'll need to copy this if we steal some kwargs for defaults
                 dict kwargs = reinterpret_borrow<dict>(kwargs_in);
 
@@ -749,6 +761,10 @@ class cpp_function : public function {
                         }
 
                         if (value) {
+                            // If we're at the py::args index then first insert a stub for it to be replaced later
+                            if (func.has_args && call.args.size() == func.nargs_pos)
+                                call.args.push_back(none());
+
                             call.args.push_back(value);
                             call.args_convert.push_back(arg_rec.convert);
                         }
@@ -771,16 +787,19 @@ class cpp_function : public function {
                         // We didn't copy out any position arguments from the args_in tuple, so we
                         // can reuse it directly without copying:
                         extra_args = reinterpret_borrow<tuple>(args_in);
-                    } else if (args_copied >= n_args_in) {
+                    } else if (positional_args_copied >= n_args_in) {
                         extra_args = tuple(0);
                     } else {
-                        size_t args_size = n_args_in - args_copied;
+                        size_t args_size = n_args_in - positional_args_copied;
                         extra_args = tuple(args_size);
                         for (size_t i = 0; i < args_size; ++i) {
-                            extra_args[i] = PyTuple_GET_ITEM(args_in, args_copied + i);
+                            extra_args[i] = PyTuple_GET_ITEM(args_in, positional_args_copied + i);
                         }
                     }
-                    call.args.push_back(extra_args);
+                    if (call.args.size() <= func.nargs_pos)
+                        call.args.push_back(extra_args);
+                    else
+                        call.args[func.nargs_pos] = extra_args;
                     call.args_convert.push_back(false);
                     call.args_ref = std::move(extra_args);
                 }

tests/test_kwargs_and_defaults.cpp

@@ -56,6 +56,23 @@ TEST_SUBMODULE(kwargs_and_defaults, m) {
     m.def("mixed_plus_args_kwargs_defaults", mixed_plus_both,
             py::arg("i") = 1, py::arg("j") = 3.14159);
 
+    m.def("args_kwonly",
+            [](int i, double j, py::args args, int z) { return py::make_tuple(i, j, args, z); },
+            "i"_a, "j"_a, "z"_a);
+    m.def("args_kwonly_kwargs",
+            [](int i, double j, py::args args, int z, py::kwargs kwargs) {
+                return py::make_tuple(i, j, args, z, kwargs); },
+            "i"_a, "j"_a, py::kw_only{}, "z"_a);
+    m.def("args_kwonly_kwargs_defaults",
+            [](int i, double j, py::args args, int z, py::kwargs kwargs) {
+                return py::make_tuple(i, j, args, z, kwargs); },
+            "i"_a = 1, "j"_a = 3.14159, "z"_a = 42);
+    m.def("args_kwonly_full_monty",
+            [](int h, int i, double j, py::args args, int z, py::kwargs kwargs) {
+                return py::make_tuple(h, i, j, args, z, kwargs); },
+            py::arg() = 1, py::arg() = 2, py::pos_only{}, "j"_a = 3.14159, "z"_a = 42);
+
+
     // test_args_refcount
     // PyPy needs a garbage collection to get the reference count values to match CPython's behaviour
     #ifdef PYPY_VERSION