Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions Include/internal/pycore_call.h
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,14 @@ PyAPI_FUNC(PyObject*) _PyObject_CallMethod(
const char *format, ...);


extern PyObject *_PyObject_VectorcallPrepend(
PyThreadState *tstate,
PyObject *callable,
PyObject *arg,
PyObject *const *args,
size_t nargsf,
PyObject *kwnames);

/* === Vectorcall protocol (PEP 590) ============================= */

// Call callable using tp_call. Arguments are like PyObject_Vectorcall(),
Expand Down
5 changes: 5 additions & 0 deletions Include/internal/pycore_function.h
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,11 @@ static inline PyObject* _PyFunction_GET_BUILTINS(PyObject *func) {
#define _PyFunction_GET_BUILTINS(func) _PyFunction_GET_BUILTINS(_PyObject_CAST(func))


/* Get the callable wrapped by a classmethod.
Returns a borrowed reference.
The caller must ensure 'cm' is a classmethod object. */
extern PyObject *_PyClassMethod_GetFunc(PyObject *cm);

/* Get the callable wrapped by a staticmethod.
Returns a borrowed reference.
The caller must ensure 'sm' is a staticmethod object. */
Expand Down
2 changes: 1 addition & 1 deletion Include/internal/pycore_object.h
Original file line number Diff line number Diff line change
Expand Up @@ -895,7 +895,7 @@ extern PyObject *_PyType_LookupRefAndVersion(PyTypeObject *, PyObject *,
extern unsigned int
_PyType_LookupStackRefAndVersion(PyTypeObject *type, PyObject *name, _PyStackRef *out);

PyAPI_FUNC(int) _PyObject_GetMethodStackRef(PyThreadState *ts, PyObject *obj,
extern int _PyObject_GetMethodStackRef(PyThreadState *ts, _PyStackRef *self,
PyObject *name, _PyStackRef *method);

// Like PyObject_GetAttr but returns a _PyStackRef. For types, this can
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Improve scaling of :func:`classmethod` and :func:`staticmethod` calls in
the free-threaded build by avoiding the descriptor ``__get__`` call.
105 changes: 88 additions & 17 deletions Objects/call.c
Original file line number Diff line number Diff line change
Expand Up @@ -828,6 +828,60 @@ object_vacall(PyThreadState *tstate, PyObject *base,
return result;
}

PyObject *
_PyObject_VectorcallPrepend(PyThreadState *tstate, PyObject *callable,
PyObject *arg, PyObject *const *args,
size_t nargsf, PyObject *kwnames)
{
Py_ssize_t nargs = PyVectorcall_NARGS(nargsf);
assert(nargs == 0 || args[nargs-1]);

PyObject *result;
if (nargsf & PY_VECTORCALL_ARGUMENTS_OFFSET) {
/* PY_VECTORCALL_ARGUMENTS_OFFSET is set, so we are allowed to mutate the vector */
PyObject **newargs = (PyObject**)args - 1;
nargs += 1;
PyObject *tmp = newargs[0];
newargs[0] = arg;
assert(newargs[nargs-1]);
result = _PyObject_VectorcallTstate(tstate, callable, newargs,
nargs, kwnames);
newargs[0] = tmp;
}
else {
Py_ssize_t nkwargs = (kwnames == NULL) ? 0 : PyTuple_GET_SIZE(kwnames);
Py_ssize_t totalargs = nargs + nkwargs;
if (totalargs == 0) {
return _PyObject_VectorcallTstate(tstate, callable, &arg, 1, NULL);
}

PyObject *newargs_stack[_PY_FASTCALL_SMALL_STACK];
PyObject **newargs;
if (totalargs <= (Py_ssize_t)Py_ARRAY_LENGTH(newargs_stack) - 1) {
newargs = newargs_stack;
}
else {
newargs = PyMem_Malloc((totalargs+1) * sizeof(PyObject *));
if (newargs == NULL) {
_PyErr_NoMemory(tstate);
return NULL;
}
}
/* use borrowed references */
newargs[0] = arg;
/* bpo-37138: since totalargs > 0, it's impossible that args is NULL.
* We need this, since calling memcpy() with a NULL pointer is
* undefined behaviour. */
assert(args != NULL);
memcpy(newargs + 1, args, totalargs * sizeof(PyObject *));
result = _PyObject_VectorcallTstate(tstate, callable,
newargs, nargs+1, kwnames);
if (newargs != newargs_stack) {
PyMem_Free(newargs);
}
}
return result;
}

PyObject *
PyObject_VectorcallMethod(PyObject *name, PyObject *const *args,
Expand All @@ -838,31 +892,44 @@ PyObject_VectorcallMethod(PyObject *name, PyObject *const *args,
assert(PyVectorcall_NARGS(nargsf) >= 1);

PyThreadState *tstate = _PyThreadState_GET();
_PyCStackRef method;
_PyCStackRef self, method;
_PyThreadState_PushCStackRef(tstate, &self);
_PyThreadState_PushCStackRef(tstate, &method);
/* Use args[0] as "self" argument */
int unbound = _PyObject_GetMethodStackRef(tstate, args[0], name, &method.ref);
if (PyStackRef_IsNull(method.ref)) {
self.ref = PyStackRef_FromPyObjectBorrow(args[0]);
int unbound = _PyObject_GetMethodStackRef(tstate, &self.ref, name, &method.ref);
if (unbound < 0) {
_PyThreadState_PopCStackRef(tstate, &method);
_PyThreadState_PopCStackRef(tstate, &self);
return NULL;
}

PyObject *callable = PyStackRef_AsPyObjectBorrow(method.ref);
PyObject *self_obj = PyStackRef_AsPyObjectBorrow(self.ref);
PyObject *result;

if (unbound) {
EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_METHOD, callable);
if (self_obj == NULL) {
/* Skip "self". We can keep PY_VECTORCALL_ARGUMENTS_OFFSET since
* args[-1] in the onward call is args[0] here. */
result = _PyObject_VectorcallTstate(tstate, callable,
args + 1, nargsf - 1, kwnames);
}
else if (self_obj == args[0]) {
/* We must remove PY_VECTORCALL_ARGUMENTS_OFFSET since
* that would be interpreted as allowing to change args[-1] */
nargsf &= ~PY_VECTORCALL_ARGUMENTS_OFFSET;
result = _PyObject_VectorcallTstate(tstate, callable, args,
nargsf & ~PY_VECTORCALL_ARGUMENTS_OFFSET,
kwnames);
}
else {
/* Skip "self". We can keep PY_VECTORCALL_ARGUMENTS_OFFSET since
* args[-1] in the onward call is args[0] here. */
args++;
nargsf--;
/* classmethod: self_obj is the type, not args[0]. Replace
* args[0] with self_obj and call the underlying callable. */
result = _PyObject_VectorcallPrepend(tstate, callable, self_obj,
args + 1, nargsf - 1, kwnames);
}
EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_METHOD, callable);
PyObject *result = _PyObject_VectorcallTstate(tstate, callable,
args, nargsf, kwnames);
_PyThreadState_PopCStackRef(tstate, &method);
_PyThreadState_PopCStackRef(tstate, &self);
return result;
}

Expand All @@ -875,22 +942,26 @@ PyObject_CallMethodObjArgs(PyObject *obj, PyObject *name, ...)
return null_error(tstate);
}

_PyCStackRef method;
_PyCStackRef self, method;
_PyThreadState_PushCStackRef(tstate, &self);
_PyThreadState_PushCStackRef(tstate, &method);
int is_method = _PyObject_GetMethodStackRef(tstate, obj, name, &method.ref);
if (PyStackRef_IsNull(method.ref)) {
self.ref = PyStackRef_FromPyObjectBorrow(obj);
int res = _PyObject_GetMethodStackRef(tstate, &self.ref, name, &method.ref);
if (res < 0) {
_PyThreadState_PopCStackRef(tstate, &method);
_PyThreadState_PopCStackRef(tstate, &self);
return NULL;
}
PyObject *callable = PyStackRef_AsPyObjectBorrow(method.ref);
obj = is_method ? obj : NULL;
PyObject *self_obj = PyStackRef_AsPyObjectBorrow(self.ref);

va_list vargs;
va_start(vargs, name);
PyObject *result = object_vacall(tstate, obj, callable, vargs);
PyObject *result = object_vacall(tstate, self_obj, callable, vargs);
va_end(vargs);

_PyThreadState_PopCStackRef(tstate, &method);
_PyThreadState_PopCStackRef(tstate, &self);
return result;
}

Expand Down
49 changes: 1 addition & 48 deletions Objects/classobject.c
Original file line number Diff line number Diff line change
Expand Up @@ -52,54 +52,7 @@ method_vectorcall(PyObject *method, PyObject *const *args,
PyThreadState *tstate = _PyThreadState_GET();
PyObject *self = PyMethod_GET_SELF(method);
PyObject *func = PyMethod_GET_FUNCTION(method);
Py_ssize_t nargs = PyVectorcall_NARGS(nargsf);
assert(nargs == 0 || args[nargs-1]);

PyObject *result;
if (nargsf & PY_VECTORCALL_ARGUMENTS_OFFSET) {
/* PY_VECTORCALL_ARGUMENTS_OFFSET is set, so we are allowed to mutate the vector */
PyObject **newargs = (PyObject**)args - 1;
nargs += 1;
PyObject *tmp = newargs[0];
newargs[0] = self;
assert(newargs[nargs-1]);
result = _PyObject_VectorcallTstate(tstate, func, newargs,
nargs, kwnames);
newargs[0] = tmp;
}
else {
Py_ssize_t nkwargs = (kwnames == NULL) ? 0 : PyTuple_GET_SIZE(kwnames);
Py_ssize_t totalargs = nargs + nkwargs;
if (totalargs == 0) {
return _PyObject_VectorcallTstate(tstate, func, &self, 1, NULL);
}

PyObject *newargs_stack[_PY_FASTCALL_SMALL_STACK];
PyObject **newargs;
if (totalargs <= (Py_ssize_t)Py_ARRAY_LENGTH(newargs_stack) - 1) {
newargs = newargs_stack;
}
else {
newargs = PyMem_Malloc((totalargs+1) * sizeof(PyObject *));
if (newargs == NULL) {
_PyErr_NoMemory(tstate);
return NULL;
}
}
/* use borrowed references */
newargs[0] = self;
/* bpo-37138: since totalargs > 0, it's impossible that args is NULL.
* We need this, since calling memcpy() with a NULL pointer is
* undefined behaviour. */
assert(args != NULL);
memcpy(newargs + 1, args, totalargs * sizeof(PyObject *));
result = _PyObject_VectorcallTstate(tstate, func,
newargs, nargs+1, kwnames);
if (newargs != newargs_stack) {
PyMem_Free(newargs);
}
}
return result;
return _PyObject_VectorcallPrepend(tstate, func, self, args, nargsf, kwnames);
}


Expand Down
8 changes: 8 additions & 0 deletions Objects/funcobject.c
Original file line number Diff line number Diff line change
Expand Up @@ -1470,6 +1470,7 @@ cm_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
if (cm == NULL) {
return NULL;
}
_PyObject_SetDeferredRefcount((PyObject *)cm);
if (cm_set_callable(cm, callable) < 0) {
Py_DECREF(cm);
return NULL;
Expand Down Expand Up @@ -1906,6 +1907,13 @@ PyStaticMethod_New(PyObject *callable)
return (PyObject *)sm;
}

PyObject *
_PyClassMethod_GetFunc(PyObject *self)
{
classmethod *cm = _PyClassMethod_CAST(self);
return cm->cm_callable;
}

PyObject *
_PyStaticMethod_GetFunc(PyObject *self)
{
Expand Down
Loading
Loading