From c6f2debc86412b693a1aa63913055ba37ed77599 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Thu, 16 Jan 2025 22:34:43 +0000 Subject: [PATCH 01/10] gh-100239: specialize bitwise logical binary ops on non-negative int --- Lib/test/test_opcache.py | 11 ++++++ ...-01-16-22-54-12.gh-issue-100239.7_HpBU.rst | 1 + Python/specialize.c | 36 +++++++++++++++++++ 3 files changed, 48 insertions(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-01-16-22-54-12.gh-issue-100239.7_HpBU.rst diff --git a/Lib/test/test_opcache.py b/Lib/test/test_opcache.py index 72b845fcc8fdbf..b4f4edab89ad07 100644 --- a/Lib/test/test_opcache.py +++ b/Lib/test/test_opcache.py @@ -1362,6 +1362,17 @@ def binary_op_add_extend(): self.assert_specialized(binary_op_add_extend, "BINARY_OP_EXTEND") self.assert_no_opcode(binary_op_add_extend, "BINARY_OP") + def binary_op_bitwise_extend(): + for _ in range(100): + a, b = 1, 2 + x = a | b + y = a & b + z = a ^ b + + binary_op_bitwise_extend() + self.assert_specialized(binary_op_bitwise_extend, "BINARY_OP_EXTEND") + self.assert_no_opcode(binary_op_bitwise_extend, "BINARY_OP") + @cpython_only @requires_specialization_ft diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-01-16-22-54-12.gh-issue-100239.7_HpBU.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-01-16-22-54-12.gh-issue-100239.7_HpBU.rst new file mode 100644 index 00000000000000..ca8006a92680df --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-01-16-22-54-12.gh-issue-100239.7_HpBU.rst @@ -0,0 +1 @@ +Specialize ``BINARY_OP`` for bitwise logical ops on non-negative ints. diff --git a/Python/specialize.c b/Python/specialize.c index 09bfcd34b5a543..a15160124b4dd7 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -2414,6 +2414,35 @@ binary_op_fail_kind(int oparg, PyObject *lhs, PyObject *rhs) /** Binary Op Specialization Extensions */ +/* long-long */ + +static inline int +is_nonnegative_compactlong(PyObject *v) +{ + return PyLong_CheckExact(v) && + (!_PyLong_IsNegative((PyLongObject *)v)) && + _PyLong_IsCompact((PyLongObject *)v); +} + +static int +nonnegative_compactlongs_guard(PyObject *lhs, PyObject *rhs) +{ + return (is_nonnegative_compactlong(lhs) && is_nonnegative_compactlong(rhs)); +} + +#define NONNEGATIVE_LONGS_ACTION(NAME, OP) \ + static PyObject * \ + (NAME)(PyObject *lhs, PyObject *rhs) \ + { \ + Py_ssize_t rhs_val = _PyLong_CompactValue((PyLongObject *)rhs); \ + Py_ssize_t lhs_val = _PyLong_CompactValue((PyLongObject *)lhs); \ + return PyLong_FromLong(lhs_val OP rhs_val); \ + } +NONNEGATIVE_LONGS_ACTION(nonnegative_compactlongs_or, |) +NONNEGATIVE_LONGS_ACTION(nonnegative_compactlongs_and, &) +NONNEGATIVE_LONGS_ACTION(nonnegative_compactlongs_xor, ^) +#undef NONNEGATIVE_LONGS_ACTION + /* float-long */ static int @@ -2466,6 +2495,12 @@ LONG_FLOAT_ACTION(compactlong_float_multiply, *) LONG_FLOAT_ACTION(compactlong_float_true_div, /) #undef LONG_FLOAT_ACTION +static _PyBinaryOpSpecializationDescr nonnegative_compactlongs_specs[NB_OPARG_LAST+1] = { + [NB_OR] = {nonnegative_compactlongs_guard, nonnegative_compactlongs_or}, + [NB_AND] = {nonnegative_compactlongs_guard, nonnegative_compactlongs_and}, + [NB_XOR] = {nonnegative_compactlongs_guard, nonnegative_compactlongs_xor}, +}; + static _PyBinaryOpSpecializationDescr float_compactlong_specs[NB_OPARG_LAST+1] = { [NB_ADD] = {float_compactlong_guard, float_compactlong_add}, [NB_SUBTRACT] = {float_compactlong_guard, float_compactlong_subtract}, @@ -2494,6 +2529,7 @@ binary_op_extended_specialization(PyObject *lhs, PyObject *rhs, int oparg, LOOKUP_SPEC(compactlong_float_specs, oparg); LOOKUP_SPEC(float_compactlong_specs, oparg); + LOOKUP_SPEC(nonnegative_compactlongs_specs, oparg); #undef LOOKUP_SPEC return 0; } From 862bc67ad7978c9ca24b3f27746be10d1901ca73 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Thu, 16 Jan 2025 23:47:57 +0000 Subject: [PATCH 02/10] the globals thing --- Tools/c-analyzer/cpython/ignored.tsv | 1 + 1 file changed, 1 insertion(+) diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 213165b06866b5..22caccd2a4ec9b 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -378,6 +378,7 @@ Python/pylifecycle.c - INTERPRETER_TRAMPOLINE_CODEDEF - Python/pystate.c - initial - Python/specialize.c - adaptive_opcodes - Python/specialize.c - cache_requirements - +Python/specialize.c - nonnegative_compactlongs_specs - Python/specialize.c - float_compactlong_specs - Python/specialize.c - compactlong_float_specs - Python/stdlib_module_names.h - _Py_stdlib_module_names - From 69e66f5e9a4a2f7e3270c68609343d6f2673d52b Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Fri, 17 Jan 2025 09:06:08 +0000 Subject: [PATCH 03/10] Update Misc/NEWS.d/next/Core_and_Builtins/2025-01-16-22-54-12.gh-issue-100239.7_HpBU.rst Co-authored-by: Pieter Eendebak --- .../2025-01-16-22-54-12.gh-issue-100239.7_HpBU.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-01-16-22-54-12.gh-issue-100239.7_HpBU.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-01-16-22-54-12.gh-issue-100239.7_HpBU.rst index ca8006a92680df..b8a4a81b1cf689 100644 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-01-16-22-54-12.gh-issue-100239.7_HpBU.rst +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-01-16-22-54-12.gh-issue-100239.7_HpBU.rst @@ -1 +1 @@ -Specialize ``BINARY_OP`` for bitwise logical ops on non-negative ints. +Specialize ``BINARY_OP`` for bitwise logical operations on non-negative compact ints. From 01734132922aced596f3448bd97f104df976c408 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Fri, 17 Jan 2025 15:03:15 +0000 Subject: [PATCH 04/10] add checks in test --- Lib/test/test_opcache.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_opcache.py b/Lib/test/test_opcache.py index b4f4edab89ad07..22aaa353f97d19 100644 --- a/Lib/test/test_opcache.py +++ b/Lib/test/test_opcache.py @@ -1364,10 +1364,13 @@ def binary_op_add_extend(): def binary_op_bitwise_extend(): for _ in range(100): - a, b = 1, 2 + a, b = 2, 7 x = a | b + self.assertEqual(x, 7) y = a & b + self.assertEqual(y, 2) z = a ^ b + self.assertEqual(z, 5) binary_op_bitwise_extend() self.assert_specialized(binary_op_bitwise_extend, "BINARY_OP_EXTEND") From 61d8bb3c48121221cc54f464252b982aa9fb3789 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Fri, 17 Jan 2025 18:08:45 +0000 Subject: [PATCH 05/10] no need to exclude negative --- Python/specialize.c | 27 +++++++++++++-------------- Tools/c-analyzer/cpython/ignored.tsv | 2 +- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/Python/specialize.c b/Python/specialize.c index a15160124b4dd7..ed678ec257108a 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -2417,20 +2417,19 @@ binary_op_fail_kind(int oparg, PyObject *lhs, PyObject *rhs) /* long-long */ static inline int -is_nonnegative_compactlong(PyObject *v) +is_compactlong(PyObject *v) { return PyLong_CheckExact(v) && - (!_PyLong_IsNegative((PyLongObject *)v)) && _PyLong_IsCompact((PyLongObject *)v); } static int -nonnegative_compactlongs_guard(PyObject *lhs, PyObject *rhs) +compactlongs_guard(PyObject *lhs, PyObject *rhs) { - return (is_nonnegative_compactlong(lhs) && is_nonnegative_compactlong(rhs)); + return (is_compactlong(lhs) && is_compactlong(rhs)); } -#define NONNEGATIVE_LONGS_ACTION(NAME, OP) \ +#define BITWISE_LONGS_ACTION(NAME, OP) \ static PyObject * \ (NAME)(PyObject *lhs, PyObject *rhs) \ { \ @@ -2438,10 +2437,10 @@ nonnegative_compactlongs_guard(PyObject *lhs, PyObject *rhs) Py_ssize_t lhs_val = _PyLong_CompactValue((PyLongObject *)lhs); \ return PyLong_FromLong(lhs_val OP rhs_val); \ } -NONNEGATIVE_LONGS_ACTION(nonnegative_compactlongs_or, |) -NONNEGATIVE_LONGS_ACTION(nonnegative_compactlongs_and, &) -NONNEGATIVE_LONGS_ACTION(nonnegative_compactlongs_xor, ^) -#undef NONNEGATIVE_LONGS_ACTION +BITWISE_LONGS_ACTION(compactlongs_or, |) +BITWISE_LONGS_ACTION(compactlongs_and, &) +BITWISE_LONGS_ACTION(compactlongs_xor, ^) +#undef BITWISE_LONGS_ACTION /* float-long */ @@ -2495,10 +2494,10 @@ LONG_FLOAT_ACTION(compactlong_float_multiply, *) LONG_FLOAT_ACTION(compactlong_float_true_div, /) #undef LONG_FLOAT_ACTION -static _PyBinaryOpSpecializationDescr nonnegative_compactlongs_specs[NB_OPARG_LAST+1] = { - [NB_OR] = {nonnegative_compactlongs_guard, nonnegative_compactlongs_or}, - [NB_AND] = {nonnegative_compactlongs_guard, nonnegative_compactlongs_and}, - [NB_XOR] = {nonnegative_compactlongs_guard, nonnegative_compactlongs_xor}, +static _PyBinaryOpSpecializationDescr compactlongs_specs[NB_OPARG_LAST+1] = { + [NB_OR] = {compactlongs_guard, compactlongs_or}, + [NB_AND] = {compactlongs_guard, compactlongs_and}, + [NB_XOR] = {compactlongs_guard, compactlongs_xor}, }; static _PyBinaryOpSpecializationDescr float_compactlong_specs[NB_OPARG_LAST+1] = { @@ -2529,7 +2528,7 @@ binary_op_extended_specialization(PyObject *lhs, PyObject *rhs, int oparg, LOOKUP_SPEC(compactlong_float_specs, oparg); LOOKUP_SPEC(float_compactlong_specs, oparg); - LOOKUP_SPEC(nonnegative_compactlongs_specs, oparg); + LOOKUP_SPEC(compactlongs_specs, oparg); #undef LOOKUP_SPEC return 0; } diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 22caccd2a4ec9b..8200eb9bbf5c7f 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -378,7 +378,7 @@ Python/pylifecycle.c - INTERPRETER_TRAMPOLINE_CODEDEF - Python/pystate.c - initial - Python/specialize.c - adaptive_opcodes - Python/specialize.c - cache_requirements - -Python/specialize.c - nonnegative_compactlongs_specs - +Python/specialize.c - compactlongs_specs - Python/specialize.c - float_compactlong_specs - Python/specialize.c - compactlong_float_specs - Python/stdlib_module_names.h - _Py_stdlib_module_names - From 1f4830099b608dd7f507c68206bcec24f758d657 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Fri, 17 Jan 2025 22:35:31 +0000 Subject: [PATCH 06/10] EXIT_IF --> DEOPT_IF --- Include/internal/pycore_opcode_metadata.h | 2 +- Include/internal/pycore_uop_metadata.h | 2 +- Python/bytecodes.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index f7c23ad634d6be..8078920b6c9ac2 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1995,7 +1995,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[266] = { [BINARY_OP_ADD_FLOAT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, [BINARY_OP_ADD_INT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, [BINARY_OP_ADD_UNICODE] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, - [BINARY_OP_EXTEND] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, + [BINARY_OP_EXTEND] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, [BINARY_OP_INPLACE_ADD_UNICODE] = { true, INSTR_FMT_IXC0000, HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG }, [BINARY_OP_MULTIPLY_FLOAT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, [BINARY_OP_MULTIPLY_INT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 298e918b872c62..cf22cd4e8a2669 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -82,7 +82,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_GUARD_BOTH_UNICODE] = HAS_EXIT_FLAG, [_BINARY_OP_ADD_UNICODE] = HAS_ERROR_FLAG | HAS_PURE_FLAG, [_BINARY_OP_INPLACE_ADD_UNICODE] = HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG, - [_GUARD_BINARY_OP_EXTEND] = HAS_EXIT_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_BINARY_OP_EXTEND] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, [_BINARY_OP_EXTEND] = HAS_ESCAPES_FLAG | HAS_PURE_FLAG, [_BINARY_SUBSCR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_BINARY_SLICE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, diff --git a/Python/bytecodes.c b/Python/bytecodes.c index b1d61a8707bd4c..e6fa13dd20a025 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -749,7 +749,7 @@ dummy_func( assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5); assert(d && d->guard); int res = d->guard(left_o, right_o); - EXIT_IF(!res); + DEOPT_IF(!res); } pure op(_BINARY_OP_EXTEND, (descr/4, left, right -- res)) { From e43ac582c7ec747b049af77808428f26ab4d6ee4 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Fri, 17 Jan 2025 23:48:37 +0000 Subject: [PATCH 07/10] add specialization failure kind stats --- Python/specialize.c | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/Python/specialize.c b/Python/specialize.c index ed678ec257108a..8dc88b24b15f8f 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -581,6 +581,10 @@ _PyCode_Quicken(_Py_CODEUNIT *instructions, Py_ssize_t size, PyObject *consts, #define SPEC_FAIL_BINARY_OP_TRUE_DIVIDE_FLOAT 26 #define SPEC_FAIL_BINARY_OP_TRUE_DIVIDE_OTHER 27 #define SPEC_FAIL_BINARY_OP_XOR 28 +#define SPEC_FAIL_BINARY_OP_OR_INT 29 +#define SPEC_FAIL_BINARY_OP_OR_DIFFERENT_TYPES 30 +#define SPEC_FAIL_BINARY_OP_XOR_INT 31 +#define SPEC_FAIL_BINARY_OP_XOR_DIFFERENT_TYPES 32 /* Calls */ @@ -2379,6 +2383,12 @@ binary_op_fail_kind(int oparg, PyObject *lhs, PyObject *rhs) return SPEC_FAIL_BINARY_OP_MULTIPLY_OTHER; case NB_OR: case NB_INPLACE_OR: + if (!Py_IS_TYPE(lhs, Py_TYPE(rhs))) { + return SPEC_FAIL_BINARY_OP_OR_DIFFERENT_TYPES; + } + if (PyLong_CheckExact(lhs)) { + return SPEC_FAIL_BINARY_OP_OR_INT; + } return SPEC_FAIL_BINARY_OP_OR; case NB_POWER: case NB_INPLACE_POWER: @@ -2406,6 +2416,12 @@ binary_op_fail_kind(int oparg, PyObject *lhs, PyObject *rhs) return SPEC_FAIL_BINARY_OP_TRUE_DIVIDE_OTHER; case NB_XOR: case NB_INPLACE_XOR: + if (!Py_IS_TYPE(lhs, Py_TYPE(rhs))) { + return SPEC_FAIL_BINARY_OP_XOR_DIFFERENT_TYPES; + } + if (PyLong_CheckExact(lhs)) { + return SPEC_FAIL_BINARY_OP_XOR_INT; + } return SPEC_FAIL_BINARY_OP_XOR; } Py_UNREACHABLE(); From 6916df41ca6e60e366263471d307a0895c844031 Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Sat, 18 Jan 2025 20:38:44 +0000 Subject: [PATCH 08/10] remove non-negative --- .../2025-01-16-22-54-12.gh-issue-100239.7_HpBU.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-01-16-22-54-12.gh-issue-100239.7_HpBU.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-01-16-22-54-12.gh-issue-100239.7_HpBU.rst index b8a4a81b1cf689..6f086b7ecc0036 100644 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-01-16-22-54-12.gh-issue-100239.7_HpBU.rst +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-01-16-22-54-12.gh-issue-100239.7_HpBU.rst @@ -1 +1 @@ -Specialize ``BINARY_OP`` for bitwise logical operations on non-negative compact ints. +Specialize ``BINARY_OP`` for bitwise logical operations on compact ints. From 6476205bd274990699e8177dd1abcbeba5e44fec Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Tue, 21 Jan 2025 12:54:58 +0000 Subject: [PATCH 09/10] add inplace ops --- Lib/test/test_opcache.py | 9 +++++++++ Python/specialize.c | 3 +++ 2 files changed, 12 insertions(+) diff --git a/Lib/test/test_opcache.py b/Lib/test/test_opcache.py index 22aaa353f97d19..cf6446567413c8 100644 --- a/Lib/test/test_opcache.py +++ b/Lib/test/test_opcache.py @@ -1371,6 +1371,15 @@ def binary_op_bitwise_extend(): self.assertEqual(y, 2) z = a ^ b self.assertEqual(z, 5) + a, b = 3, 9 + a |= b + self.assertEqual(a, 11) + a, b = 11, 9 + a &= b + self.assertEqual(a, 9) + a, b = 3, 9 + a ^= b + self.assertEqual(a, 10) binary_op_bitwise_extend() self.assert_specialized(binary_op_bitwise_extend, "BINARY_OP_EXTEND") diff --git a/Python/specialize.c b/Python/specialize.c index 8dc88b24b15f8f..7bfe81cf788e93 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -2514,6 +2514,9 @@ static _PyBinaryOpSpecializationDescr compactlongs_specs[NB_OPARG_LAST+1] = { [NB_OR] = {compactlongs_guard, compactlongs_or}, [NB_AND] = {compactlongs_guard, compactlongs_and}, [NB_XOR] = {compactlongs_guard, compactlongs_xor}, + [NB_INPLACE_OR] = {compactlongs_guard, compactlongs_or}, + [NB_INPLACE_AND] = {compactlongs_guard, compactlongs_and}, + [NB_INPLACE_XOR] = {compactlongs_guard, compactlongs_xor}, }; static _PyBinaryOpSpecializationDescr float_compactlong_specs[NB_OPARG_LAST+1] = { From 634b388270f7544faed82351304ba8cffc644d3d Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Tue, 21 Jan 2025 19:30:27 +0000 Subject: [PATCH 10/10] PyLong_FromLong --> PyLong_FromSsize_t --- Python/specialize.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/specialize.c b/Python/specialize.c index 413356cf873996..46f005c5146b2a 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -2495,7 +2495,7 @@ compactlongs_guard(PyObject *lhs, PyObject *rhs) { \ Py_ssize_t rhs_val = _PyLong_CompactValue((PyLongObject *)rhs); \ Py_ssize_t lhs_val = _PyLong_CompactValue((PyLongObject *)lhs); \ - return PyLong_FromLong(lhs_val OP rhs_val); \ + return PyLong_FromSsize_t(lhs_val OP rhs_val); \ } BITWISE_LONGS_ACTION(compactlongs_or, |) BITWISE_LONGS_ACTION(compactlongs_and, &)