bpo-36974: implement PEP 590 (GH-13185)
Co-authored-by: Jeroen Demeyer <J.Demeyer@UGent.be>
Co-authored-by: Mark Shannon <mark@hotpy.org>
diff --git a/Objects/call.c b/Objects/call.c
index b608492..183a5c2 100644
--- a/Objects/call.c
+++ b/Objects/call.c
@@ -5,6 +5,10 @@
#include "frameobject.h"
+static PyObject *
+cfunction_call_varargs(PyObject *func, PyObject *args, PyObject *kwargs);
+
+
int
_PyObject_HasFastCall(PyObject *callable)
{
@@ -83,131 +87,132 @@
/* --- Core PyObject call functions ------------------------------- */
PyObject *
-_PyObject_FastCallDict(PyObject *callable, PyObject *const *args, Py_ssize_t nargs,
- PyObject *kwargs)
+_PyObject_FastCallDict(PyObject *callable, PyObject *const *args,
+ size_t nargsf, PyObject *kwargs)
{
/* _PyObject_FastCallDict() must not be called with an exception set,
because it can clear it (directly or indirectly) and so the
caller loses its exception */
assert(!PyErr_Occurred());
-
assert(callable != NULL);
+
+ Py_ssize_t nargs = PyVectorcall_NARGS(nargsf);
assert(nargs >= 0);
assert(nargs == 0 || args != NULL);
assert(kwargs == NULL || PyDict_Check(kwargs));
- if (PyFunction_Check(callable)) {
- return _PyFunction_FastCallDict(callable, args, nargs, kwargs);
+ vectorcallfunc func = _PyVectorcall_Function(callable);
+ if (func == NULL) {
+ /* Use tp_call instead */
+ return _PyObject_MakeTpCall(callable, args, nargs, kwargs);
}
- else if (PyCFunction_Check(callable)) {
- return _PyCFunction_FastCallDict(callable, args, nargs, kwargs);
+
+ PyObject *res;
+ if (kwargs == NULL) {
+ res = func(callable, args, nargsf, NULL);
}
else {
- PyObject *argstuple, *result;
- ternaryfunc call;
-
- /* Slow-path: build a temporary tuple */
- call = callable->ob_type->tp_call;
- if (call == NULL) {
- PyErr_Format(PyExc_TypeError, "'%.200s' object is not callable",
- callable->ob_type->tp_name);
+ PyObject *kwnames;
+ PyObject *const *newargs;
+ if (_PyStack_UnpackDict(args, nargs, kwargs, &newargs, &kwnames) < 0) {
return NULL;
}
-
- argstuple = _PyTuple_FromArray(args, nargs);
- if (argstuple == NULL) {
- return NULL;
+ res = func(callable, newargs, nargs, kwnames);
+ if (kwnames != NULL) {
+ Py_ssize_t i, n = PyTuple_GET_SIZE(kwnames) + nargs;
+ for (i = 0; i < n; i++) {
+ Py_DECREF(newargs[i]);
+ }
+ PyMem_Free((PyObject **)newargs);
+ Py_DECREF(kwnames);
}
-
- if (Py_EnterRecursiveCall(" while calling a Python object")) {
- Py_DECREF(argstuple);
- return NULL;
- }
-
- result = (*call)(callable, argstuple, kwargs);
-
- Py_LeaveRecursiveCall();
- Py_DECREF(argstuple);
-
- result = _Py_CheckFunctionResult(callable, result, NULL);
- return result;
}
+ return _Py_CheckFunctionResult(callable, res, NULL);
}
PyObject *
-_PyObject_FastCallKeywords(PyObject *callable, PyObject *const *stack, Py_ssize_t nargs,
- PyObject *kwnames)
+_PyObject_MakeTpCall(PyObject *callable, PyObject *const *args, Py_ssize_t nargs, PyObject *keywords)
{
- /* _PyObject_FastCallKeywords() must not be called with an exception set,
- because it can clear it (directly or indirectly) and so the
- caller loses its exception */
- assert(!PyErr_Occurred());
+ /* Slow path: build a temporary tuple for positional arguments and a
+ * temporary dictionary for keyword arguments (if any) */
+ ternaryfunc call = Py_TYPE(callable)->tp_call;
+ if (call == NULL) {
+ PyErr_Format(PyExc_TypeError, "'%.200s' object is not callable",
+ Py_TYPE(callable)->tp_name);
+ return NULL;
+ }
assert(nargs >= 0);
- assert(kwnames == NULL || PyTuple_CheckExact(kwnames));
-
- /* kwnames must only contains str strings, no subclass, and all keys must
- be unique: these checks are implemented in Python/ceval.c and
- _PyArg_ParseStackAndKeywords(). */
-
- if (PyFunction_Check(callable)) {
- return _PyFunction_FastCallKeywords(callable, stack, nargs, kwnames);
+ assert(nargs == 0 || args != NULL);
+ assert(keywords == NULL || PyTuple_Check(keywords) || PyDict_Check(keywords));
+ PyObject *argstuple = _PyTuple_FromArray(args, nargs);
+ if (argstuple == NULL) {
+ return NULL;
}
- if (PyCFunction_Check(callable)) {
- return _PyCFunction_FastCallKeywords(callable, stack, nargs, kwnames);
+
+ PyObject *kwdict;
+ if (keywords == NULL || PyDict_Check(keywords)) {
+ kwdict = keywords;
}
else {
- /* Slow-path: build a temporary tuple for positional arguments and a
- temporary dictionary for keyword arguments (if any) */
-
- ternaryfunc call;
- PyObject *argstuple;
- PyObject *kwdict, *result;
- Py_ssize_t nkwargs;
-
- nkwargs = (kwnames == NULL) ? 0 : PyTuple_GET_SIZE(kwnames);
- assert((nargs == 0 && nkwargs == 0) || stack != NULL);
-
- call = callable->ob_type->tp_call;
- if (call == NULL) {
- PyErr_Format(PyExc_TypeError, "'%.200s' object is not callable",
- callable->ob_type->tp_name);
- return NULL;
- }
-
- argstuple = _PyTuple_FromArray(stack, nargs);
- if (argstuple == NULL) {
- return NULL;
- }
-
- if (nkwargs > 0) {
- kwdict = _PyStack_AsDict(stack + nargs, kwnames);
+ if (PyTuple_GET_SIZE(keywords)) {
+ assert(args != NULL);
+ kwdict = _PyStack_AsDict(args + nargs, keywords);
if (kwdict == NULL) {
Py_DECREF(argstuple);
return NULL;
}
}
else {
- kwdict = NULL;
+ keywords = kwdict = NULL;
}
-
- if (Py_EnterRecursiveCall(" while calling a Python object")) {
- Py_DECREF(argstuple);
- Py_XDECREF(kwdict);
- return NULL;
- }
-
- result = (*call)(callable, argstuple, kwdict);
-
- Py_LeaveRecursiveCall();
-
- Py_DECREF(argstuple);
- Py_XDECREF(kwdict);
-
- result = _Py_CheckFunctionResult(callable, result, NULL);
- return result;
}
+
+ PyObject *result = NULL;
+ if (Py_EnterRecursiveCall(" while calling a Python object") == 0)
+ {
+ result = call(callable, argstuple, kwdict);
+ Py_LeaveRecursiveCall();
+ }
+
+ Py_DECREF(argstuple);
+ if (kwdict != keywords) {
+ Py_DECREF(kwdict);
+ }
+
+ result = _Py_CheckFunctionResult(callable, result, NULL);
+ return result;
+}
+
+
+PyObject *
+PyVectorcall_Call(PyObject *callable, PyObject *tuple, PyObject *kwargs)
+{
+ vectorcallfunc func = _PyVectorcall_Function(callable);
+ if (func == NULL) {
+ PyErr_Format(PyExc_TypeError, "'%.200s' object does not support vectorcall",
+ Py_TYPE(callable)->tp_name);
+ return NULL;
+ }
+ PyObject *const *args;
+ Py_ssize_t nargs = PyTuple_GET_SIZE(tuple);
+ PyObject *kwnames;
+ if (_PyStack_UnpackDict(_PyTuple_ITEMS(tuple), nargs,
+ kwargs, &args, &kwnames) < 0) {
+ return NULL;
+ }
+ PyObject *result = func(callable, args, nargs, kwnames);
+ if (kwnames != NULL) {
+ Py_ssize_t i, n = PyTuple_GET_SIZE(kwnames) + nargs;
+ for (i = 0; i < n; i++) {
+ Py_DECREF(args[i]);
+ }
+ PyMem_Free((PyObject **)args);
+ Py_DECREF(kwnames);
+ }
+
+ return result;
}
@@ -224,14 +229,13 @@
assert(PyTuple_Check(args));
assert(kwargs == NULL || PyDict_Check(kwargs));
- if (PyFunction_Check(callable)) {
- return _PyFunction_FastCallDict(callable,
- _PyTuple_ITEMS(args),
- PyTuple_GET_SIZE(args),
- kwargs);
+ if (_PyVectorcall_Function(callable) != NULL) {
+ return PyVectorcall_Call(callable, args, kwargs);
}
else if (PyCFunction_Check(callable)) {
- return PyCFunction_Call(callable, args, kwargs);
+ /* This must be a METH_VARARGS function, otherwise we would be
+ * in the previous case */
+ return cfunction_call_varargs(callable, args, kwargs);
}
else {
call = callable->ob_type->tp_call;
@@ -384,9 +388,10 @@
return result;
}
+
PyObject *
-_PyFunction_FastCallKeywords(PyObject *func, PyObject *const *stack,
- Py_ssize_t nargs, PyObject *kwnames)
+_PyFunction_FastCallKeywords(PyObject *func, PyObject* const* stack,
+ size_t nargsf, PyObject *kwnames)
{
PyCodeObject *co = (PyCodeObject *)PyFunction_GET_CODE(func);
PyObject *globals = PyFunction_GET_GLOBALS(func);
@@ -397,6 +402,7 @@
Py_ssize_t nd;
assert(PyFunction_Check(func));
+ Py_ssize_t nargs = PyVectorcall_NARGS(nargsf);
assert(nargs >= 0);
assert(kwnames == NULL || PyTuple_CheckExact(kwnames));
assert((nargs == 0 && nkwargs == 0) || stack != NULL);
@@ -725,13 +731,14 @@
PyObject *
_PyCFunction_FastCallKeywords(PyObject *func,
- PyObject *const *args, Py_ssize_t nargs,
+ PyObject *const *args, size_t nargsf,
PyObject *kwnames)
{
PyObject *result;
assert(func != NULL);
assert(PyCFunction_Check(func));
+ Py_ssize_t nargs = PyVectorcall_NARGS(nargsf);
result = _PyMethodDef_RawFastCallKeywords(((PyCFunctionObject*)func)->m_ml,
PyCFunction_GET_SELF(func),
@@ -751,6 +758,7 @@
PyObject *self = PyCFunction_GET_SELF(func);
PyObject *result;
+ assert(PyCFunction_GET_FLAGS(func) & METH_VARARGS);
if (PyCFunction_GET_FLAGS(func) & METH_KEYWORDS) {
if (Py_EnterRecursiveCall(" while calling a Python object")) {
return NULL;
@@ -783,18 +791,12 @@
PyObject *
PyCFunction_Call(PyObject *func, PyObject *args, PyObject *kwargs)
{
- /* first try METH_VARARGS to pass directly args tuple unchanged.
- _PyMethodDef_RawFastCallDict() creates a new temporary tuple
- for METH_VARARGS. */
+ /* For METH_VARARGS, we cannot use vectorcall as the vectorcall pointer
+ * is NULL. This is intentional, since vectorcall would be slower. */
if (PyCFunction_GET_FLAGS(func) & METH_VARARGS) {
return cfunction_call_varargs(func, args, kwargs);
}
- else {
- return _PyCFunction_FastCallDict(func,
- _PyTuple_ITEMS(args),
- PyTuple_GET_SIZE(args),
- kwargs);
- }
+ return PyVectorcall_Call(func, args, kwargs);
}
diff --git a/Objects/classobject.c b/Objects/classobject.c
index 1ee8978..cfc2446 100644
--- a/Objects/classobject.c
+++ b/Objects/classobject.c
@@ -40,6 +40,45 @@
return ((PyMethodObject *)im)->im_self;
}
+
+static PyObject *
+method_vectorcall(PyObject *method, PyObject *const *args,
+ size_t nargsf, PyObject *kwnames)
+{
+ assert(Py_TYPE(method) == &PyMethod_Type);
+ PyObject *self, *func, *result;
+ self = PyMethod_GET_SELF(method);
+ func = PyMethod_GET_FUNCTION(method);
+ Py_ssize_t nargs = PyVectorcall_NARGS(nargsf);
+
+ if (nargsf & PY_VECTORCALL_ARGUMENTS_OFFSET) {
+ /* PY_VECTORCALL_ARGUMENTS_OFFSET is set, so we are allowed to mutate the vector */
+ PyObject **newargs = (PyObject**)args - 1;
+ nargs += 1;
+ PyObject *tmp = newargs[0];
+ newargs[0] = self;
+ result = _PyObject_Vectorcall(func, newargs, nargs, kwnames);
+ newargs[0] = tmp;
+ }
+ else {
+ Py_ssize_t nkwargs = (kwnames == NULL) ? 0 : PyTuple_GET_SIZE(kwnames);
+ PyObject **newargs;
+ Py_ssize_t totalargs = nargs + nkwargs;
+ newargs = PyMem_Malloc((totalargs+1) * sizeof(PyObject *));
+ if (newargs == NULL) {
+ PyErr_NoMemory();
+ return NULL;
+ }
+ /* use borrowed references */
+ newargs[0] = self;
+ memcpy(newargs + 1, args, totalargs * sizeof(PyObject *));
+ result = _PyObject_Vectorcall(func, newargs, nargs+1, kwnames);
+ PyMem_Free(newargs);
+ }
+ return result;
+}
+
+
/* Method objects are used for bound instance methods returned by
instancename.methodname. ClassName.methodname returns an ordinary
function.
@@ -69,6 +108,7 @@
im->im_func = func;
Py_XINCREF(self);
im->im_self = self;
+ im->vectorcall = method_vectorcall;
_PyObject_GC_TRACK(im);
return (PyObject *)im;
}
@@ -309,7 +349,7 @@
sizeof(PyMethodObject),
0,
(destructor)method_dealloc, /* tp_dealloc */
- 0, /* tp_print */
+ offsetof(PyMethodObject, vectorcall), /* tp_vectorcall_offset */
0, /* tp_getattr */
0, /* tp_setattr */
0, /* tp_reserved */
@@ -323,7 +363,8 @@
method_getattro, /* tp_getattro */
PyObject_GenericSetAttr, /* tp_setattro */
0, /* tp_as_buffer */
- Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */
+ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC |
+ _Py_TPFLAGS_HAVE_VECTORCALL, /* tp_flags */
method_doc, /* tp_doc */
(traverseproc)method_traverse, /* tp_traverse */
0, /* tp_clear */
diff --git a/Objects/descrobject.c b/Objects/descrobject.c
index 6c99f9b..7590185 100644
--- a/Objects/descrobject.c
+++ b/Objects/descrobject.c
@@ -265,13 +265,14 @@
// same to methoddescr_call(), but use FASTCALL convention.
PyObject *
_PyMethodDescr_FastCallKeywords(PyObject *descrobj,
- PyObject *const *args, Py_ssize_t nargs,
+ PyObject *const *args, size_t nargsf,
PyObject *kwnames)
{
assert(Py_TYPE(descrobj) == &PyMethodDescr_Type);
PyMethodDescrObject *descr = (PyMethodDescrObject *)descrobj;
PyObject *self, *result;
+ Py_ssize_t nargs = PyVectorcall_NARGS(nargsf);
/* Make sure that the first argument is acceptable as 'self' */
if (nargs < 1) {
PyErr_Format(PyExc_TypeError,
@@ -542,7 +543,7 @@
sizeof(PyMethodDescrObject),
0,
(destructor)descr_dealloc, /* tp_dealloc */
- 0, /* tp_print */
+ offsetof(PyMethodDescrObject, vectorcall), /* tp_vectorcall_offset */
0, /* tp_getattr */
0, /* tp_setattr */
0, /* tp_reserved */
@@ -557,6 +558,7 @@
0, /* tp_setattro */
0, /* tp_as_buffer */
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC |
+ _Py_TPFLAGS_HAVE_VECTORCALL |
Py_TPFLAGS_METHOD_DESCRIPTOR, /* tp_flags */
0, /* tp_doc */
descr_traverse, /* tp_traverse */
@@ -752,8 +754,10 @@
descr = (PyMethodDescrObject *)descr_new(&PyMethodDescr_Type,
type, method->ml_name);
- if (descr != NULL)
+ if (descr != NULL) {
descr->d_method = method;
+ descr->vectorcall = &_PyMethodDescr_FastCallKeywords;
+ }
return (PyObject *)descr;
}
diff --git a/Objects/funcobject.c b/Objects/funcobject.c
index fb7abfa..2b1f42d 100644
--- a/Objects/funcobject.c
+++ b/Objects/funcobject.c
@@ -36,6 +36,7 @@
op->func_defaults = NULL; /* No default arguments */
op->func_kwdefaults = NULL; /* No keyword only defaults */
op->func_closure = NULL;
+ op->vectorcall = _PyFunction_FastCallKeywords;
consts = ((PyCodeObject *)code)->co_consts;
if (PyTuple_Size(consts) >= 1) {
@@ -649,7 +650,7 @@
sizeof(PyFunctionObject),
0,
(destructor)func_dealloc, /* tp_dealloc */
- 0, /* tp_print */
+ offsetof(PyFunctionObject, vectorcall), /* tp_vectorcall_offset */
0, /* tp_getattr */
0, /* tp_setattr */
0, /* tp_reserved */
@@ -664,6 +665,7 @@
0, /* tp_setattro */
0, /* tp_as_buffer */
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC |
+ _Py_TPFLAGS_HAVE_VECTORCALL |
Py_TPFLAGS_METHOD_DESCRIPTOR, /* tp_flags */
func_new__doc__, /* tp_doc */
(traverseproc)func_traverse, /* tp_traverse */
diff --git a/Objects/methodobject.c b/Objects/methodobject.c
index 9fed3fc..76497c9 100644
--- a/Objects/methodobject.c
+++ b/Objects/methodobject.c
@@ -46,6 +46,14 @@
op->m_self = self;
Py_XINCREF(module);
op->m_module = module;
+ if (ml->ml_flags & METH_VARARGS) {
+ /* For METH_VARARGS functions, it's more efficient to use tp_call
+ * instead of vectorcall. */
+ op->vectorcall = NULL;
+ }
+ else {
+ op->vectorcall = &_PyCFunction_FastCallKeywords;
+ }
_PyObject_GC_TRACK(op);
return (PyObject *)op;
}
@@ -264,7 +272,7 @@
sizeof(PyCFunctionObject),
0,
(destructor)meth_dealloc, /* tp_dealloc */
- 0, /* tp_print */
+ offsetof(PyCFunctionObject, vectorcall), /* tp_vectorcall_offset */
0, /* tp_getattr */
0, /* tp_setattr */
0, /* tp_reserved */
@@ -278,7 +286,8 @@
PyObject_GenericGetAttr, /* tp_getattro */
0, /* tp_setattro */
0, /* tp_as_buffer */
- Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC,/* tp_flags */
+ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC |
+ _Py_TPFLAGS_HAVE_VECTORCALL, /* tp_flags */
0, /* tp_doc */
(traverseproc)meth_traverse, /* tp_traverse */
0, /* tp_clear */