Skip to content

bpo-42927: Inline cache for slots #24216

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 7 commits into from
Jan 30, 2021
Merged

bpo-42927: Inline cache for slots #24216

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
aca2939
Implement inline cache for slots
gvanrossum Jan 14, 2021
e334e53
Comment out debug print
gvanrossum Jan 15, 2021
ec0a590
Fall through to slow path if slot is NULL
gvanrossum Jan 15, 2021
b725a65
📜🤖 Added by blurb_it.
blurb-it[bot] Jan 15, 2021
4c2eb69
Fix the buildbot test failure (I hope).
gvanrossum Jan 28, 2021
17c8a95
Merge remote-tracking branch 'origin/master' into slotscache
gvanrossum Jan 28, 2021
86c4b39
Improve comment
gvanrossum Jan 28, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions Misc/NEWS.d/next/Core and Builtins/2021-01-15-20-05-56.bpo-42927.GI-l-7.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
The inline cache for ``LOAD_ATTR`` now also optimizes access to attributes defined by ``__slots__``.
This makes reading such attribute up to 30% faster.
180 changes: 116 additions & 64 deletions Python/ceval.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
#include "opcode.h"
#include "pydtrace.h"
#include "setobject.h"
#include "structmember.h" // struct PyMemberDef, T_OFFSET_EX

#include <ctype.h>

Expand Down Expand Up @@ -3169,114 +3170,165 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, PyFrameObject *f, int throwflag)
if (co_opcache != NULL && PyType_HasFeature(type, Py_TPFLAGS_VALID_VERSION_TAG))
{
if (co_opcache->optimized > 0) {
/* Fast path -- cache hit makes LOAD_ATTR ~30% faster */
// Fast path -- cache hit makes LOAD_ATTR ~30% faster.
la = &co_opcache->u.la;
if (la->type == type && la->tp_version_tag == type->tp_version_tag)
{
assert(type->tp_dict != NULL);
assert(type->tp_dictoffset > 0);

dictptr = (PyObject **) ((char *)owner + type->tp_dictoffset);
dict = *dictptr;
if (dict != NULL && PyDict_CheckExact(dict)) {
Py_ssize_t hint = la->hint;
Py_INCREF(dict);
res = NULL;
la->hint = _PyDict_GetItemHint((PyDictObject*)dict, name, hint, &res);

// Hint >= 0 is a dict index; hint == -1 is a dict miss.
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Oh, right! How did I miss that on review :/

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same way I missed it writing the code. :-(

This code is hairy.

// Hint < -1 is an inverted slot offset: offset is strictly > 0,
// so ~offset is strictly < -1 (assuming 2's complement).
if (la->hint < -1) {
// Even faster path -- slot hint.
Py_ssize_t offset = ~la->hint;
// fprintf(stderr, "Using hint for offset %zd\n", offset);
char *addr = (char *)owner + offset;
res = *(PyObject **)addr;
if (res != NULL) {
if (la->hint == hint && hint >= 0) {
/* Our hint has helped -- cache hit. */
OPCACHE_STAT_ATTR_HIT();
} else {
/* The hint we provided didn't work.
Maybe next time? */
OPCACHE_MAYBE_DEOPT_LOAD_ATTR();
}

Py_INCREF(res);
SET_TOP(res);
Py_DECREF(owner);
Py_DECREF(dict);
DISPATCH();
}
// Else slot is NULL. Fall through to slow path to raise AttributeError(name).
// Don't DEOPT, since the slot is still there.
} else {
// Fast path for dict.
assert(type->tp_dict != NULL);
assert(type->tp_dictoffset > 0);

dictptr = (PyObject **) ((char *)owner + type->tp_dictoffset);
dict = *dictptr;
if (dict != NULL && PyDict_CheckExact(dict)) {
Py_ssize_t hint = la->hint;
Py_INCREF(dict);
res = NULL;
la->hint = _PyDict_GetItemHint((PyDictObject*)dict, name, hint, &res);

if (res != NULL) {
if (la->hint == hint && hint >= 0) {
// Our hint has helped -- cache hit.
OPCACHE_STAT_ATTR_HIT();
} else {
// The hint we provided didn't work.
// Maybe next time?
OPCACHE_MAYBE_DEOPT_LOAD_ATTR();
}

Py_INCREF(res);
SET_TOP(res);
Py_DECREF(owner);
Py_DECREF(dict);
DISPATCH();
} else {
// This attribute can be missing sometimes;
// we don't want to optimize this lookup.
OPCACHE_DEOPT_LOAD_ATTR();
Py_DECREF(dict);
}
} else {
// This attribute can be missing sometimes -- we
// don't want to optimize this lookup.
// There is no dict, or __dict__ doesn't satisfy PyDict_CheckExact.
OPCACHE_DEOPT_LOAD_ATTR();
Py_DECREF(dict);
}
} else {
// There is no dict, or __dict__ doesn't satisfy PyDict_CheckExact
OPCACHE_DEOPT_LOAD_ATTR();
}
} else {
// The type of the object has either been updated,
// or is different. Maybe it will stabilize?
OPCACHE_MAYBE_DEOPT_LOAD_ATTR();
}

OPCACHE_STAT_ATTR_MISS();
}

if (co_opcache != NULL && /* co_opcache can be NULL after a DEOPT() call. */
if (co_opcache != NULL && // co_opcache can be NULL after a DEOPT() call.
type->tp_getattro == PyObject_GenericGetAttr)
{
Py_ssize_t ret;

if (type->tp_dictoffset > 0) {
if (type->tp_dict == NULL) {
if (PyType_Ready(type) < 0) {
Py_DECREF(owner);
SET_TOP(NULL);
goto error;
}
if (type->tp_dict == NULL) {
if (PyType_Ready(type) < 0) {
Py_DECREF(owner);
SET_TOP(NULL);
goto error;
}
if (_PyType_Lookup(type, name) == NULL) {
dictptr = (PyObject **) ((char *)owner + type->tp_dictoffset);
dict = *dictptr;
}
PyObject *descr = _PyType_Lookup(type, name);
if (descr != NULL) {
// We found an attribute with a data-like descriptor.
PyTypeObject *dtype = Py_TYPE(descr);
if (dtype == &PyMemberDescr_Type) { // It's a slot
PyMemberDescrObject *member = (PyMemberDescrObject *)descr;
struct PyMemberDef *dmem = member->d_member;
if (dmem->type == T_OBJECT_EX) {
Py_ssize_t offset = dmem->offset;
assert(offset > 0); // 0 would be confused with dict hint == -1 (miss).

if (co_opcache->optimized == 0) {
// First time we optimize this opcode.
OPCACHE_STAT_ATTR_OPT();
co_opcache->optimized = OPCODE_CACHE_MAX_TRIES;
// fprintf(stderr, "Setting hint for %s, offset %zd\n", dmem->name, offset);
}

if (dict != NULL && PyDict_CheckExact(dict)) {
Py_INCREF(dict);
res = NULL;
ret = _PyDict_GetItemHint((PyDictObject*)dict, name, -1, &res);
la = &co_opcache->u.la;
la->type = type;
la->tp_version_tag = type->tp_version_tag;
la->hint = ~offset;

char *addr = (char *)owner + offset;
res = *(PyObject **)addr;
if (res != NULL) {
Py_INCREF(res);
Py_DECREF(dict);
Py_DECREF(owner);
SET_TOP(res);

if (co_opcache->optimized == 0) {
// First time we optimize this opcode. */
OPCACHE_STAT_ATTR_OPT();
co_opcache->optimized = OPCODE_CACHE_MAX_TRIES;
}

la = &co_opcache->u.la;
la->type = type;
la->tp_version_tag = type->tp_version_tag;
la->hint = ret;

DISPATCH();
}
// Else slot is NULL. Fall through to slow path to raise AttributeError(name).
}
// Else it's a slot of a different type. We don't handle those.
}
// Else it's some other kind of descriptor that we don't handle.
OPCACHE_DEOPT_LOAD_ATTR();
} else if (type->tp_dictoffset > 0) {
// We found an instance with a __dict__.
dictptr = (PyObject **) ((char *)owner + type->tp_dictoffset);
dict = *dictptr;

if (dict != NULL && PyDict_CheckExact(dict)) {
Py_INCREF(dict);
res = NULL;
Py_ssize_t hint = _PyDict_GetItemHint((PyDictObject*)dict, name, -1, &res);
if (res != NULL) {
Py_INCREF(res);
Py_DECREF(dict);
} else {
// There is no dict, or __dict__ doesn't satisfy PyDict_CheckExact
OPCACHE_DEOPT_LOAD_ATTR();
Py_DECREF(owner);
SET_TOP(res);

if (co_opcache->optimized == 0) {
// First time we optimize this opcode.
OPCACHE_STAT_ATTR_OPT();
co_opcache->optimized = OPCODE_CACHE_MAX_TRIES;
}

la = &co_opcache->u.la;
la->type = type;
la->tp_version_tag = type->tp_version_tag;
la->hint = hint;

DISPATCH();
}
Py_DECREF(dict);
} else {
// We failed to find an attribute without a data-like descriptor
// There is no dict, or __dict__ doesn't satisfy PyDict_CheckExact.
OPCACHE_DEOPT_LOAD_ATTR();
}
} else {
// The object's class does not have a tp_dictoffset we can use
// The object's class does not have a tp_dictoffset we can use.
OPCACHE_DEOPT_LOAD_ATTR();
}
} else if (type->tp_getattro != PyObject_GenericGetAttr) {
OPCACHE_DEOPT_LOAD_ATTR();
}
}

/* slow path */
// Slow path.
res = PyObject_GetAttr(owner, name);
Py_DECREF(owner);
SET_TOP(res);
Expand Down