diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 3c126aaef1187a..e2cde0a23d8f7b 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -192,6 +192,9 @@ void _PyObject_ClearInstanceAttributes(PyObject *self); void _PyObject_FreeInstanceAttributes(PyObject *self); int _PyObject_IsInstanceDictEmpty(PyObject *); +PyObject *_PyType_FindNameInMRO(PyTypeObject *type, PyObject *name, int *error, + Py_ssize_t *mro_index); + #ifdef __cplusplus } #endif diff --git a/Include/opcode.h b/Include/opcode.h index 87ed32c019909a..72a113e27c6a9e 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -160,23 +160,24 @@ extern "C" { #define LOAD_ATTR_WITH_HINT 81 #define LOAD_ATTR_SLOT 87 #define LOAD_ATTR_MODULE 88 -#define LOAD_GLOBAL_ADAPTIVE 122 -#define LOAD_GLOBAL_MODULE 123 -#define LOAD_GLOBAL_BUILTIN 127 -#define LOAD_METHOD_ADAPTIVE 128 -#define LOAD_METHOD_CACHED 134 -#define LOAD_METHOD_CLASS 140 -#define LOAD_METHOD_MODULE 143 -#define LOAD_METHOD_NO_DICT 149 -#define STORE_ATTR_ADAPTIVE 150 -#define STORE_ATTR_INSTANCE_VALUE 151 -#define STORE_ATTR_SLOT 153 -#define STORE_ATTR_WITH_HINT 154 -#define LOAD_FAST__LOAD_FAST 158 -#define STORE_FAST__LOAD_FAST 159 -#define LOAD_FAST__LOAD_CONST 167 -#define LOAD_CONST__LOAD_FAST 168 -#define STORE_FAST__STORE_FAST 169 +#define LOAD_ATTR_CLASS 122 +#define LOAD_GLOBAL_ADAPTIVE 123 +#define LOAD_GLOBAL_MODULE 127 +#define LOAD_GLOBAL_BUILTIN 128 +#define LOAD_METHOD_ADAPTIVE 134 +#define LOAD_METHOD_CACHED 140 +#define LOAD_METHOD_CLASS 143 +#define LOAD_METHOD_MODULE 149 +#define LOAD_METHOD_NO_DICT 150 +#define STORE_ATTR_ADAPTIVE 151 +#define STORE_ATTR_INSTANCE_VALUE 153 +#define STORE_ATTR_SLOT 154 +#define STORE_ATTR_WITH_HINT 158 +#define LOAD_FAST__LOAD_FAST 159 +#define STORE_FAST__LOAD_FAST 167 +#define LOAD_FAST__LOAD_CONST 168 +#define LOAD_CONST__LOAD_FAST 169 +#define STORE_FAST__STORE_FAST 170 #define DO_TRACING 255 #ifdef NEED_OPCODE_JUMP_TABLES static uint32_t _PyOpcode_RelativeJump[8] = { diff --git a/Lib/opcode.py b/Lib/opcode.py index 66d5ca766bec64..3b638fcb91fdfa 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -244,6 +244,7 @@ def jabs_op(name, op): "LOAD_ATTR_WITH_HINT", "LOAD_ATTR_SLOT", "LOAD_ATTR_MODULE", + "LOAD_ATTR_CLASS", "LOAD_GLOBAL_ADAPTIVE", "LOAD_GLOBAL_MODULE", "LOAD_GLOBAL_BUILTIN", diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-10-22-15-20-43.bpo-45565.rQvNzI.rst b/Misc/NEWS.d/next/Core and Builtins/2021-10-22-15-20-43.bpo-45565.rQvNzI.rst new file mode 100644 index 00000000000000..3c9a55540c86c1 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2021-10-22-15-20-43.bpo-45565.rQvNzI.rst @@ -0,0 +1,2 @@ +Improve ``LOAD_METHOD_CLASS`` specialization by adding more specialized +forms. diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 18bea476ea9f25..4618e028b74601 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -3776,8 +3776,9 @@ _PyType_GetModuleByDef(PyTypeObject *type, struct PyModuleDef *def) /* Internal API to look for a name through the MRO, bypassing the method cache. This returns a borrowed reference, and might set an exception. 'error' is set to: -1: error with exception; 1: error without exception; 0: ok */ -static PyObject * -find_name_in_mro(PyTypeObject *type, PyObject *name, int *error) +PyObject * +_PyType_FindNameInMRO(PyTypeObject *type, PyObject *name, int *error, + Py_ssize_t *mro_index) { Py_ssize_t i, n; PyObject *mro, *res, *base, *dict; @@ -3830,6 +3831,9 @@ find_name_in_mro(PyTypeObject *type, PyObject *name, int *error) } } *error = 0; + if (mro_index != NULL) { + *mro_index = i; + } done: Py_DECREF(mro); return res; @@ -3858,7 +3862,7 @@ _PyType_Lookup(PyTypeObject *type, PyObject *name) /* We may end up clearing live exceptions below, so make sure it's ours. */ assert(!PyErr_Occurred()); - res = find_name_in_mro(type, name, &error); + res = _PyType_FindNameInMRO(type, name, &error, NULL); /* Only put NULL results into cache if there was no error. */ if (error) { /* It's not ideal to clear the error condition, @@ -8330,7 +8334,7 @@ update_one_slot(PyTypeObject *type, slotdef *p) assert(!PyErr_Occurred()); do { /* Use faster uncached lookup as we won't get any cache hits during type setup. */ - descr = find_name_in_mro(type, p->name_strobj, &error); + descr = _PyType_FindNameInMRO(type, p->name_strobj, &error, NULL); if (descr == NULL) { if (error == -1) { /* It is unlikely but not impossible that there has been an exception diff --git a/Python/ceval.c b/Python/ceval.c index 4ac0b53dd6e466..6f4375fb42e538 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -3750,6 +3750,40 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DISPATCH(); } + TARGET(LOAD_ATTR_CLASS) { + assert(cframe.use_tracing == 0); + PyObject *owner = TOP(); + PyObject *res; + SpecializedCacheEntry *caches = GET_CACHE(); + _PyAdaptiveEntry *cache0 = &caches[0].adaptive; + _PyAttrCache *cache1 = &caches[-1].attr; + _PyObjectCache *cache2 = &caches[-2].obj; + assert(cache1->tp_version != 0); + DEOPT_IF(!PyType_Check(owner), LOAD_ATTR); + PyTypeObject *tp = (PyTypeObject *)owner; + + DEOPT_IF(tp->tp_mro != cache2->obj, LOAD_ATTR); + PyTypeObject *real_owner = (PyTypeObject *)PyTuple_GET_ITEM(tp->tp_mro, + cache0->index); + assert(PyType_Check(real_owner)); + PyDictObject *dict = (PyDictObject *)real_owner->tp_dict; + assert(dict != NULL); + assert(PyDict_CheckExact((PyObject *)dict)); + DEOPT_IF(dict->ma_keys->dk_version != cache1->tp_version, LOAD_ATTR); + PyObject *name = GETITEM(names, cache0->original_oparg); + uint32_t hint = cache1->dk_version_or_hint; + DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, LOAD_ATTR); + PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint; + DEOPT_IF(ep->me_key != name, LOAD_ATTR); + res = ep->me_value; + DEOPT_IF(res == NULL, LOAD_ATTR); + STAT_INC(LOAD_ATTR, hit); + Py_INCREF(res); + SET_TOP(res); + Py_DECREF(owner); + DISPATCH(); + } + TARGET(STORE_ATTR_ADAPTIVE) { assert(cframe.use_tracing == 0); SpecializedCacheEntry *cache = GET_CACHE(); diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 93a9e4ab42cfbc..70b5063c8a8e81 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -121,44 +121,44 @@ static void *opcode_targets[256] = { &&TARGET_RERAISE, &&TARGET_COPY, &&TARGET_JUMP_IF_NOT_EXC_MATCH, + &&TARGET_LOAD_ATTR_CLASS, &&TARGET_LOAD_GLOBAL_ADAPTIVE, - &&TARGET_LOAD_GLOBAL_MODULE, &&TARGET_LOAD_FAST, &&TARGET_STORE_FAST, &&TARGET_DELETE_FAST, + &&TARGET_LOAD_GLOBAL_MODULE, &&TARGET_LOAD_GLOBAL_BUILTIN, - &&TARGET_LOAD_METHOD_ADAPTIVE, &&TARGET_GEN_START, &&TARGET_RAISE_VARARGS, &&TARGET_CALL_FUNCTION, &&TARGET_MAKE_FUNCTION, &&TARGET_BUILD_SLICE, - &&TARGET_LOAD_METHOD_CACHED, + &&TARGET_LOAD_METHOD_ADAPTIVE, &&TARGET_MAKE_CELL, &&TARGET_LOAD_CLOSURE, &&TARGET_LOAD_DEREF, &&TARGET_STORE_DEREF, &&TARGET_DELETE_DEREF, - &&TARGET_LOAD_METHOD_CLASS, + &&TARGET_LOAD_METHOD_CACHED, &&TARGET_CALL_FUNCTION_KW, &&TARGET_CALL_FUNCTION_EX, - &&TARGET_LOAD_METHOD_MODULE, + &&TARGET_LOAD_METHOD_CLASS, &&TARGET_EXTENDED_ARG, &&TARGET_LIST_APPEND, &&TARGET_SET_ADD, &&TARGET_MAP_ADD, &&TARGET_LOAD_CLASSDEREF, + &&TARGET_LOAD_METHOD_MODULE, &&TARGET_LOAD_METHOD_NO_DICT, &&TARGET_STORE_ATTR_ADAPTIVE, - &&TARGET_STORE_ATTR_INSTANCE_VALUE, &&TARGET_MATCH_CLASS, + &&TARGET_STORE_ATTR_INSTANCE_VALUE, &&TARGET_STORE_ATTR_SLOT, - &&TARGET_STORE_ATTR_WITH_HINT, &&TARGET_FORMAT_VALUE, &&TARGET_BUILD_CONST_KEY_MAP, &&TARGET_BUILD_STRING, + &&TARGET_STORE_ATTR_WITH_HINT, &&TARGET_LOAD_FAST__LOAD_FAST, - &&TARGET_STORE_FAST__LOAD_FAST, &&TARGET_LOAD_METHOD, &&TARGET_CALL_METHOD, &&TARGET_LIST_EXTEND, @@ -166,6 +166,7 @@ static void *opcode_targets[256] = { &&TARGET_DICT_MERGE, &&TARGET_DICT_UPDATE, &&TARGET_CALL_METHOD_KW, + &&TARGET_STORE_FAST__LOAD_FAST, &&TARGET_LOAD_FAST__LOAD_CONST, &&TARGET_LOAD_CONST__LOAD_FAST, &&TARGET_STORE_FAST__STORE_FAST, @@ -253,6 +254,5 @@ static void *opcode_targets[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&_unknown_opcode, &&TARGET_DO_TRACING }; diff --git a/Python/specialize.c b/Python/specialize.c index 162728314e1003..681be6994d9fb4 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -9,6 +9,7 @@ #include // rand() + /* For guidance on adding or extending families of instructions see * ./adaptive.md */ @@ -243,7 +244,7 @@ static uint8_t adaptive_opcodes[256] = { /* The number of cache entries required for a "family" of instructions. */ static uint8_t cache_requirements[256] = { - [LOAD_ATTR] = 2, /* _PyAdaptiveEntry and _PyAttrCache */ + [LOAD_ATTR] = 3, /* _PyAdaptiveEntry, _PyAttrCache and _PyObjectCache*/ [LOAD_GLOBAL] = 2, /* _PyAdaptiveEntry and _PyLoadGlobalCache */ [LOAD_METHOD] = 3, /* _PyAdaptiveEntry, _PyAttrCache and _PyObjectCache */ [BINARY_ADD] = 0, @@ -688,13 +689,87 @@ specialize_dict_access( } } +#if COLLECT_SPECIALIZATION_STATS_DETAILED +static int load_method_fail_kind(DesciptorClassification kind); +#endif + +static int +specialize_class_load_attr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name, + _PyAdaptiveEntry *cache0, _PyAttrCache *cache1, _PyObjectCache *cache2) +{ + PyTypeObject *tp = (PyTypeObject *)owner; + Py_ssize_t mro_index; + int error; + if (tp->tp_dict == NULL) { + if (PyType_Ready(tp) < 0) { + return 1; + } + } + PyObject *descr = NULL; + DesciptorClassification kind = analyze_descriptor(tp, name, &descr, 0); + switch (kind) { + case METHOD: + case NON_DESCRIPTOR: + break; + default: + SPECIALIZATION_FAIL(LOAD_METHOD, load_method_fail_kind(kind)); + return 1; + } + if (!PyType_HasFeature(tp, Py_TPFLAGS_VALID_VERSION_TAG)) { + return 1; + } + PyObject *res = _PyType_FindNameInMRO(tp, name, &error, &mro_index); + if (res == NULL || error) { + if (error == -1) { + PyErr_Clear(); + } + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_EXPECTED_ERROR); + return 1; + } + if (mro_index != (uint16_t)mro_index) { + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_RANGE); + return 1; + } + assert(tp->tp_mro != NULL); + assert(PyTuple_CheckExact(tp->tp_mro)); + assert(mro_index >= 0); + assert(mro_index < PyTuple_GET_SIZE(tp->tp_mro)); + + PyObject *real_owner = PyTuple_GET_ITEM(tp->tp_mro, mro_index); + assert(PyType_Check(real_owner)); + PyDictObject *real_owner_dict = (PyDictObject *)((PyTypeObject *)real_owner)->tp_dict; + PyObject *value = NULL; + Py_ssize_t hint = _PyDict_GetItemHint(real_owner_dict, name, -1, &value); + assert(hint != DKIX_ERROR); + assert(hint != DKIX_EMPTY); + if (hint != (uint32_t)hint) { + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_RANGE); + return -1; + } + uint32_t keys_version = _PyDictKeys_GetVersionForCurrentState( + real_owner_dict->ma_keys); + if (keys_version == 0) { + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_VERSIONS); + return -1; + } + cache0->index = (uint16_t)mro_index; + cache1->dk_version_or_hint = (uint32_t)hint; + cache1->tp_version = keys_version; + cache2->obj = tp->tp_mro; + *instr = _Py_MAKECODEUNIT(LOAD_ATTR_CLASS, _Py_OPARG(*instr)); + return 0; +} + int _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name, SpecializedCacheEntry *cache) { _PyAdaptiveEntry *cache0 = &cache->adaptive; _PyAttrCache *cache1 = &cache[-1].attr; + _PyObjectCache *cache2 = &cache[-2].obj; + + int err; if (PyModule_CheckExact(owner)) { - int err = specialize_module_load_attr(owner, instr, name, cache0, cache1, + err = specialize_module_load_attr(owner, instr, name, cache0, cache1, LOAD_ATTR, LOAD_ATTR_MODULE); if (err) { goto fail; @@ -707,6 +782,13 @@ _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name, Sp return -1; } } + if (PyType_Check(owner)) { + err = specialize_class_load_attr(owner, instr, name, cache0, cache1, cache2); + if (err) { + goto fail; + } + goto success; + } PyObject *descr; DesciptorClassification kind = analyze_descriptor(type, name, &descr, 0); switch(kind) { @@ -764,7 +846,7 @@ _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name, Sp case ABSENT: break; } - int err = specialize_dict_access( + err = specialize_dict_access( owner, instr, type, kind, name, cache0, cache1, LOAD_ATTR, LOAD_ATTR_INSTANCE_VALUE, LOAD_ATTR_WITH_HINT );